Passed
Push — master ( 648c95...abce15 )
by Gaetano
07:02
created

MassMigrateCommand::onSubProcessOutput()   B

Complexity

Conditions 7
Paths 12

Size

Total Lines 23
Code Lines 14

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 0
CRAP Score 56

Importance

Changes 0
Metric Value
cc 7
eloc 14
nc 12
nop 3
dl 0
loc 23
ccs 0
cts 13
cp 0
crap 56
rs 8.8333
c 0
b 0
f 0
1
<?php
2
3
namespace Kaliop\eZMigrationBundle\Command;
4
5
use Symfony\Component\Console\Input\ArrayInput;
6
use Symfony\Component\Console\Input\InputOption;
7
use Symfony\Component\Console\Input\InputInterface;
8
use Symfony\Component\Console\Output\Output;
9
use Symfony\Component\Console\Output\OutputInterface;
10
use Symfony\Component\Process\ProcessBuilder;
11
use Symfony\Component\Process\PhpExecutableFinder;
12
use Kaliop\eZMigrationBundle\API\Value\Migration;
13
use Kaliop\eZMigrationBundle\API\Value\MigrationDefinition;
14
use Kaliop\eZMigrationBundle\Core\Helper\ProcessManager;
15
16
class MassMigrateCommand extends MigrateCommand
17
{
18
    const COMMAND_NAME = 'kaliop:migration:mass_migrate';
19
20
    protected $migrationsDone = array(0, 0, 0);
21
    protected $migrationsAlreadyDone = array();
22
23
    /**
24
     * @todo (!important) can we rename the option --separate-process ?
25
     */
26
    protected function configure()
27 78
    {
28
        parent::configure();
29 78
30
        $this
31
            ->setName(self::COMMAND_NAME)
32 78
            ->setAliases(array())
33 78
            ->setDescription('Executes available migration definitions, using parallelism.')
34 78
            ->addOption('concurrency', 'r', InputOption::VALUE_REQUIRED, "The number of executors to run in parallel", 2)
35 78
            ->setHelp(<<<EOT
36 78
This command is designed to scan recursively a directory for migration files and execute them all in parallel.
37 78
One child process will be spawned for each subdirectory found.
38
The maximum number of processes to run in parallel is specified via the 'concurrency' option.
39
<info>NB: this command does not guarantee that any given migration will be executed before another. Take care about dependencies.</info>
40
<info>NB: the rule that each migration filename has to be unique still applies, even if migrations are spread across different directories.</info>
41
Unlike for the 'normal' migration command, it is not recommended to use the <info>--separate-process</info> option, as it will make execution slower if you have many migrations
42
EOT
43
            )
44
        ;
45
    }
46 78
47
    /**
48
     * Execute the command.
49
     *
50
     * @param InputInterface $input
51
     * @param OutputInterface $output
52
     * @return null|int null or 0 if everything went fine, or an error code
53
     */
54
    protected function execute(InputInterface $input, OutputInterface $output)
55
    {
56
        $start = microtime(true);
57
58
        $this->setOutput($output);
59
        $this->setVerbosity($output->getVerbosity());
60
61
        $isChild = $input->getOption('child');
62
63
        if ($isChild) {
64
            $this->setVerbosity(self::VERBOSITY_CHILD);
65
        }
66
67
        $this->getContainer()->get('ez_migration_bundle.step_executed_listener.tracing')->setOutput($output);
68
69
        // q: is it worth declaring a new, dedicated migration service ?
70
        $migrationService = $this->getMigrationService();
71
        $migrationService->setLoader($this->getContainer()->get('ez_migration_bundle.loader.filesystem_recursive'));
72
73
        $force = $input->getOption('force');
74
75
        $toExecute = $this->buildMigrationsList($input->getOption('path'), $migrationService, $force, $isChild);
0 ignored issues
show
Bug introduced by
It seems like $input->getOption('path') can also be of type string[]; however, parameter $paths of Kaliop\eZMigrationBundle...::buildMigrationsList() does only seem to accept string, maybe add an additional type check? ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

75
        $toExecute = $this->buildMigrationsList(/** @scrutinizer ignore-type */ $input->getOption('path'), $migrationService, $force, $isChild);
Loading history...
76
77
        if (!count($toExecute)) {
78
            $this->writeln('<info>No migrations to execute</info>');
79
            return 0;
80
        }
81
82
        if (!$isChild) {
83
            return $this->executeAsParent($input, $output, $toExecute, $start);
84
85
        } else {
86
            return $this->executeAsChild($input, $output, $toExecute, $force, $migrationService);
87
        }
88
    }
89
90
    /**
91
     * @param InputInterface $input
92
     * @param OutputInterface $output
93
     * @param MigrationDefinition[] $toExecute
94
     * @param float $start
95
     * @return int
96
     */
97
    protected function executeAsParent($input, $output, $toExecute, $start)
98
    {
99
        $paths = $this->groupMigrationsByPath($toExecute);
100
        $this->printMigrationsList($toExecute, $input, $output, $paths);
101
102
        // ask user for confirmation to make changes
103
        if (!$this->askForConfirmation($input, $output, null)) {
104
            return 0;
105
        }
106
107
        $concurrency = $input->getOption('concurrency');
108
        $this->writeln("Executing migrations using " . count($paths) . " processes with a concurrency of $concurrency");
109
110
        $builder = new ProcessBuilder();
111
        $executableFinder = new PhpExecutableFinder();
112
        if (false !== ($php = $executableFinder->find())) {
113
            $builder->setPrefix($php);
114
        }
115
116
        // mandatory args and options
117
        $builderArgs = $this->createChildProcessArgs($input);
118
119
        $processes = array();
120
        /** @var MigrationDefinition $migrationDefinition */
121
        foreach($paths as $path => $count) {
122
            $this->writeln("<info>Queueing processing of: $path ($count migrations)</info>", OutputInterface::VERBOSITY_VERBOSE);
123
124
            $process = $builder
125
                ->setArguments(array_merge($builderArgs, array('--path=' . $path)))
126
                ->getProcess();
127
128
            $this->writeln('<info>Command: ' . $process->getCommandLine() . '</info>', OutputInterface::VERBOSITY_VERBOSE);
129
130
            // allow long migrations processes by default
131
            $process->setTimeout($this->subProcessTimeout);
132
            // allow forcing handling of sigchild. Useful on eg. Debian and Ubuntu
133
            if ($input->getOption('force-sigchild-handling')) {
134
                $process->setEnhanceSigchildCompatibility(true);
135
            }
136
            $processes[] = $process;
137
        }
138
139
        $this->writeln("Starting queued processes...");
140
141
        $this->migrationsDone = array(0, 0, 0);
142
143
        $processManager = new ProcessManager();
144
        $processManager->runParallel($processes, $concurrency, 500, array($this, 'onSubProcessOutput'));
145
146
        $failed = 0;
147
        foreach ($processes as $i => $process) {
148
            if (!$process->isSuccessful()) {
149
                $errorOutput = $process->getErrorOutput();
150
                if ($errorOutput === '') {
151
                    $errorOutput = "(separate process used to execute migration failed with no stderr output. Its exit code was: " . $process->getExitCode();
152
                    if ($process->getExitCode() == -1) {
153
                        $errorOutput .= ". If you are using Debian or Ubuntu linux, please consider using the --force-sigchild-handling option.";
154
                    }
155
                    $errorOutput .= ")";
156
                }
157
                /// @todo should we always add the exit code, even when $errorOutput is not null ?
158
                $this->errOutput->writeln("\n<error>Subprocess $i failed! Reason: " . $errorOutput . "</error>\n");
159
                $failed++;
160
            }
161
        }
162
163
        if ($input->getOption('clear-cache')) {
164
            $command = $this->getApplication()->find('cache:clear');
165
            $inputArray = new ArrayInput(array('command' => 'cache:clear'));
166
            $command->run($inputArray, $output);
167
        }
168
169
        $time = microtime(true) - $start;
170
171
        $this->writeln('<info>'.$this->migrationsDone[0].' migrations executed, '.$this->migrationsDone[1].($failed ? ' or more' : '').' failed, '.$this->migrationsDone[2].' skipped</info>');
172
173
        // since we use subprocesses, we can not measure max memory used
174
        $this->writeln("Time taken: ".sprintf('%.2f', $time)." secs");
175
176
        return $failed + $this->migrationsDone[1];
177
    }
178
179
    /**
180
     * @param InputInterface $input
181
     * @param OutputInterface $output
182
     * @param MigrationDefinition[] $toExecute
183
     * @param bool $force
184
     * @param $migrationService
185
     * @return int
186
     */
187
    protected function executeAsChild($input, $output, $toExecute, $force, $migrationService)
0 ignored issues
show
Unused Code introduced by
The parameter $output is not used and could be removed. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-unused  annotation

187
    protected function executeAsChild($input, /** @scrutinizer ignore-unused */ $output, $toExecute, $force, $migrationService)

This check looks for parameters that have been defined for a function or method, but which are not used in the method body.

Loading history...
188
    {
189
        // @todo disable signal slots that are harmful during migrations, if any
190
191
        if ($input->getOption('separate-process')) {
192
            $builder = new ProcessBuilder();
193
            $executableFinder = new PhpExecutableFinder();
194
            if (false !== $php = $executableFinder->find()) {
195
                $builder->setPrefix($php);
196
            }
197
198
            $builderArgs = parent::createChildProcessArgs($input);
199
        }
200
201
        $forceSigChild = $input->getOption('force-sigchild-handling');
202
203
        $failed = 0;
204
        $executed = 0;
205
        $skipped = 0;
206
        $total = count($toExecute);
207
208
        foreach ($toExecute as  $name => $migrationDefinition) {
209
            // let's skip migrations that we know are invalid - user was warned and he decided to proceed anyway
210
            if ($migrationDefinition->status == MigrationDefinition::STATUS_INVALID) {
211
                $this->writeln("<comment>Skipping migration (invalid definition?) Path: ".$migrationDefinition->path."</comment>", self::VERBOSITY_CHILD);
0 ignored issues
show
Bug introduced by
self::VERBOSITY_CHILD of type double is incompatible with the type integer expected by parameter $verbosity of Kaliop\eZMigrationBundle...tractCommand::writeln(). ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

211
                $this->writeln("<comment>Skipping migration (invalid definition?) Path: ".$migrationDefinition->path."</comment>", /** @scrutinizer ignore-type */ self::VERBOSITY_CHILD);
Loading history...
212
                $skipped++;
213
                continue;
214
            }
215
216
            if ($input->getOption('separate-process')) {
217
218
                try {
219
                    $this->executeMigrationInSeparateProcess($migrationDefinition, $migrationService, $builder, $builderArgs, false, $forceSigChild);
0 ignored issues
show
Comprehensibility Best Practice introduced by
The variable $builderArgs does not seem to be defined for all execution paths leading up to this point.
Loading history...
Comprehensibility Best Practice introduced by
The variable $builder does not seem to be defined for all execution paths leading up to this point.
Loading history...
220
221
                    $executed++;
222
                } catch (\Exception $e) {
223
224
                    $errorMessage = preg_replace('/^\n*Migration aborted! Reason: */', '', $e->getMessage());
225
226
                    if ($input->getOption('ignore-failures')) {
227
                        $this->errOutput->writeln("\n<error>Migration failed! Path: " . $migrationDefinition->path . ", Reason: " . $errorMessage . "</error>\n");
228
                        $failed++;
229
                        continue;
230
                    }
231
                    $this->errOutput->writeln("\n<error>Migration aborted! Path: " . $migrationDefinition->path . ", Reason: " . $errorMessage . "</error>");
232
233
                    $missed = $total - $executed - $failed - $skipped;
234
                    $this->writeln("Migrations executed: $executed, failed: $failed, skipped: $skipped, to do: $missed");
235
236
                    return 1;
237
                }
238
239
            } else {
240
241
                try {
242
                    $this->executeMigrationInProcess($migrationDefinition, $force, $migrationService, $input);
243
244
                    $executed++;
245
                } catch(\Exception $e) {
246
                    $failed++;
247
                    if ($input->getOption('ignore-failures')) {
248
                        $this->writeErrorln("<error>Migration failed! Path: " . $migrationDefinition->path . ", Reason: " . $e->getMessage() . "</error>", self::VERBOSITY_CHILD);
249
                        continue;
250
                    }
251
252
                    $this->writeErrorln("<error>Migration aborted! Path: " . $migrationDefinition->path . ", Reason: " . $e->getMessage() . "</error>", self::VERBOSITY_CHILD);
253
254
                    $missed = $total - $executed - $failed - $skipped;
255
                    $this->writeln("Migrations executed: $executed, failed: $failed, skipped: $skipped, to do: $missed");
256
257
                    return 1;
258
                }
259
260
            }
261
        }
262
263
        $this->writeln("Migrations executed: $executed, failed: $failed, skipped: $skipped", self::VERBOSITY_CHILD);
264
265
        // We do not return an error code > 0 if migrations fail, but only on proper fatals.
266
        // The parent will analyze the output of the child process to gather the number of executed/failed migrations anyway
267
        //return $failed;
268
    }
269
270
    public function onSubProcessOutput($type, $buffer, $process=null)
271
    {
272
        $lines = explode("\n", trim($buffer));
273
274
        foreach ($lines as $line) {
275
            if (preg_match('/Migrations executed: ([0-9]+), failed: ([0-9]+), skipped: ([0-9]+)/', $line, $matches)) {
276
                $this->migrationsDone[0] += $matches[1];
277
                $this->migrationsDone[1] += $matches[2];
278
                $this->migrationsDone[2] += $matches[3];
279
280
                // swallow these lines unless we are in verbose mode
281
                if ($this->verbosity <= Output::VERBOSITY_NORMAL) {
282
                    return;
283
                }
284
            }
285
286
            // we tag the output from the different processes
287
            if (trim($line) !== '') {
288
                $msg = '[' . ($process ? $process->getPid() : ''). '] ' . trim($line);
289
                if ($type == 'err') {
290
                    $this->errOutput->writeln($msg, OutputInterface::OUTPUT_RAW);
291
                } else {
292
                    $this->output->writeln($msg, OutputInterface::OUTPUT_RAW);
293
                }
294
            }
295
        }
296
    }
297
298
    /**
299
     * @param string $paths
300
     * @param $migrationService
301
     * @param bool $force
302
     * @param bool $isChild when not in child mode, do not waste time parsing migrations
303
     * @return MigrationDefinition[] parsed or unparsed, depending on
304
     *
305
     * @todo this does not scale well with many definitions or migrations
306
     */
307
    protected function buildMigrationsList($paths, $migrationService, $force = false, $isChild = false)
308
    {
309
        $migrationDefinitions = $migrationService->getMigrationsDefinitions($paths);
310
        $migrations = $migrationService->getMigrations();
311
312
        $this->migrationsAlreadyDone = array(Migration::STATUS_DONE => 0, Migration::STATUS_FAILED => 0, Migration::STATUS_SKIPPED => 0, Migration::STATUS_STARTED => 0);
313
314
        $allowedStatuses = array(Migration::STATUS_TODO);
315
        if ($force) {
316
            $allowedStatuses = array_merge($allowedStatuses, array(Migration::STATUS_DONE, Migration::STATUS_FAILED, Migration::STATUS_SKIPPED));
317
        }
318
319
        // filter away all migrations except 'to do' ones
320
        $toExecute = array();
321
        foreach($migrationDefinitions as $name => $migrationDefinition) {
322
            if (!isset($migrations[$name]) || (($migration = $migrations[$name]) && in_array($migration->status, $allowedStatuses))) {
323
                $toExecute[$name] = $isChild ? $migrationService->parseMigrationDefinition($migrationDefinition) : $migrationDefinition;
324
            }
325
            // save the list of non-executable migrations as well (even when using 'force')
326
            if (!$isChild && isset($migrations[$name]) && (($migration = $migrations[$name]) && $migration->status != Migration::STATUS_TODO)) {
327
                $this->migrationsAlreadyDone[$migration->status]++;
328
            }
329
        }
330
331
        // if user wants to execute 'all' migrations: look for some which are registered in the database even if not
332
        // found by the loader
333
        if (empty($paths)) {
334
            foreach ($migrations as $migration) {
335
                if (in_array($migration->status, $allowedStatuses) && !isset($toExecute[$migration->name])) {
336
                    $migrationDefinitions = $migrationService->getMigrationsDefinitions(array($migration->path));
337
                    if (count($migrationDefinitions)) {
338
                        $migrationDefinition = reset($migrationDefinitions);
339
                        $toExecute[$migration->name] = $isChild ? $migrationService->parseMigrationDefinition($migrationDefinition) : $migrationDefinition;
340
                    } else {
341
                        // q: shall we raise a warning here ?
342
                    }
343
                }
344
            }
345
        }
346
347
        ksort($toExecute);
348
349
        return $toExecute;
350
    }
351
352
    /**
353
     * We use a more compact output when there are *many* migrations
354
     * @param MigrationDefinition[] $toExecute
355
     * @param array $paths
356
     * @param InputInterface $input
357
     * @param OutputInterface $output
358
     */
359
    protected function printMigrationsList($toExecute, InputInterface $input, OutputInterface $output, $paths = array())
360
    {
361
        $output->writeln('Found ' . count($toExecute) . ' migrations in ' . count($paths) . ' directories');
362
        $output->writeln('In the same directories, migrations previously executed: ' . $this->migrationsAlreadyDone[Migration::STATUS_DONE] .
363
            ', failed: ' . $this->migrationsAlreadyDone[Migration::STATUS_FAILED] . ', skipped: '. $this->migrationsAlreadyDone[Migration::STATUS_SKIPPED]);
364
        if ($this->migrationsAlreadyDone[Migration::STATUS_STARTED]) {
365
            $output->writeln('<info>In the same directories, migrations currently executing: ' . $this->migrationsAlreadyDone[Migration::STATUS_STARTED] . '</info>');
366
        }
367
    }
368
369
    /**
370
     * @param MigrationDefinition[] $toExecute
371
     * @return array key: folder name, value: number of migrations found
372
     */
373
    protected function groupMigrationsByPath($toExecute)
374
    {
375
        $paths = array();
376
        foreach($toExecute as $name => $migrationDefinition) {
377
            $path = dirname($migrationDefinition->path);
378
            if (!isset($paths[$path])) {
379
                $paths[$path] = 1;
380
            } else {
381
                $paths[$path]++;
382
            }
383
        }
384
385
        ksort($paths);
386
387
        return $paths;
388
    }
389
390
    /**
391
     * Returns the command-line arguments needed to execute a migration in a separate subprocess (omitting 'path')
392
     * @param InputInterface $input
393
     * @return array
394
     */
395
    protected function createChildProcessArgs(InputInterface $input)
396
    {
397
        $kernel = $this->getContainer()->get('kernel');
398
399
        // mandatory args and options
400
        $builderArgs = array(
401
            $_SERVER['argv'][0], // sf console
402
            self::COMMAND_NAME, // name of sf command. Can we get it from the Application instead of hardcoding?
403
            '--env=' . $kernel-> getEnvironment(), // sf env
404
            '--child'
405
        );
406
        // sf/ez env options
407
        if (!$kernel->isDebug()) {
408
            $builderArgs[] = '--no-debug';
409
        }
410
        if ($input->getOption('siteaccess')) {
411
            $builderArgs[] = '--siteaccess=' . $input->getOption('siteaccess');
412
        }
413
        // 'optional' options
414
        // note: options 'clear-cache', 'no-interaction', 'path' we never propagate
415
        if ($input->getOption('admin-login')) {
416
            $builderArgs[] = '--admin-login=' . $input->getOption('admin-login');
417
        }
418
        if ($input->getOption('default-language')) {
419
            $builderArgs[] = '--default-language=' . $input->getOption('default-language');
420
        }
421
        if ($input->getOption('force')) {
422
            $builderArgs[] = '--force';
423
        }
424
        if ($input->getOption('ignore-failures')) {
425
            $builderArgs[] = '--ignore-failures';
426
        }
427
        if ($input->getOption('no-transactions')) {
428
            $builderArgs[] = '--no-transactions';
429
        }
430
        if ($input->getOption('separate-process')) {
431
            $builderArgs[] = '--separate-process';
432
        }
433
        // useful in case the subprocess has a migration step of type process/run
434
        if ($input->getOption('force-sigchild-handling')) {
435
            $builderArgs[] = '--force-sigchild-handling';
436
        }
437
438
        return $builderArgs;
439
    }
440
}
441