Completed
Branch master (e35419)
by Gaetano
06:40
created

MassMigrateCommand::executeAsChild()   B

Complexity

Conditions 10
Paths 24

Size

Total Lines 72
Code Lines 42

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 0
CRAP Score 110

Importance

Changes 0
Metric Value
cc 10
eloc 42
nc 24
nop 5
dl 0
loc 72
ccs 0
cts 40
cp 0
crap 110
rs 7.6666
c 0
b 0
f 0

How to fix   Long Method    Complexity   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
<?php
2
3
namespace Kaliop\eZMigrationBundle\Command;
4
5
use Symfony\Component\Console\Input\ArrayInput;
6
use Symfony\Component\Console\Input\InputOption;
7
use Symfony\Component\Console\Input\InputInterface;
8
use Symfony\Component\Console\Output\Output;
9
use Symfony\Component\Console\Output\OutputInterface;
10
use Kaliop\eZMigrationBundle\API\Value\Migration;
11
use Kaliop\eZMigrationBundle\API\Value\MigrationDefinition;
12
use Symfony\Component\Process\ProcessBuilder;
13
use Symfony\Component\Process\PhpExecutableFinder;
14
use Kaliop\eZMigrationBundle\Core\Helper\ProcessManager;
15
use Symfony\Component\Console\Question\ConfirmationQuestion;
16
17
class MassMigrateCommand extends MigrateCommand
18
{
19
    const COMMAND_NAME = 'kaliop:migration:mass_migrate';
20
21
    protected $migrationsDone = array(0, 0, 0);
22
    protected $migrationsAlreadyDone = array();
23
24
    /**
25
     * @todo (!important) can we rename the option --separate-process ?
26
     */
27 78
    protected function configure()
28
    {
29 78
        parent::configure();
30
31
        $this
32 78
            ->setName(self::COMMAND_NAME)
33 78
            ->setAliases(array())
34 78
            ->setDescription('Executes available migration definitions, using parallelism.')
35 78
            ->addOption('concurrency', 'r', InputOption::VALUE_REQUIRED, "The number of executors to run in parallel", 2)
36 78
            ->setHelp(<<<EOT
37 78
This command is designed to scan recursively a directory for migration files and execute them all in parallel.
38
One child process will be spawned for each subdirectory found.
39
The maximum number of processes to run in parallel is specified via the 'concurrency' option.
40
<info>NB: this command does not guarantee that any given migration will be executed before another. Take care about dependencies.</info>
41
<info>NB: the rule that each migration filename has to be unique still applies, even if migrations are spread across different directories.</info>
42
Unlike for the 'normal' migration command, it is not recommended to use the <info>--separate-process</info> option, as it will make execution much slower
43
EOT
44
            )
45
        ;
46 78
    }
47
48
    /**
49
     * Execute the command.
50
     *
51
     * @param InputInterface $input
52
     * @param OutputInterface $output
53
     * @return null|int null or 0 if everything went fine, or an error code
54
     */
55
    protected function execute(InputInterface $input, OutputInterface $output)
56
    {
57
        $start = microtime(true);
58
59
        $this->setOutput($output);
60
        $this->setVerbosity($output->getVerbosity());
61
62
        $isChild = $input->getOption('child');
63
64
        if ($isChild) {
65
            $this->setVerbosity(self::VERBOSITY_CHILD);
66
        }
67
68
        $this->getContainer()->get('ez_migration_bundle.step_executed_listener.tracing')->setOutput($output);
69
70
        // q: is it worth declaring a new, dedicated migration service ?
71
        $migrationService = $this->getMigrationService();
72
        $migrationService->setLoader($this->getContainer()->get('ez_migration_bundle.loader.filesystem_recursive'));
73
74
        $force = $input->getOption('force');
75
76
        $toExecute = $this->buildMigrationsList($input->getOption('path'), $migrationService, $force, $isChild);
0 ignored issues
show
Bug introduced by
It seems like $input->getOption('path') can also be of type string[]; however, parameter $paths of Kaliop\eZMigrationBundle...::buildMigrationsList() does only seem to accept string, maybe add an additional type check? ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

76
        $toExecute = $this->buildMigrationsList(/** @scrutinizer ignore-type */ $input->getOption('path'), $migrationService, $force, $isChild);
Loading history...
77
78
        if (!count($toExecute)) {
79
            $this->writeln('<info>No migrations to execute</info>');
80
            return 0;
81
        }
82
83
        if (!$isChild) {
84
            return $this->executeAsParent($input, $output, $toExecute, $start);
85
86
        } else {
87
            return $this->executeAsChild($input, $output, $toExecute, $force, $migrationService);
88
        }
89
    }
90
91
    protected function executeAsParent($input, $output, $toExecute, $start)
92
    {
93
        $paths = $this->groupMigrationsByPath($toExecute);
94
        $this->printMigrationsList($toExecute, $input, $output, $paths);
95
96
        // ask user for confirmation to make changes
97
        if (!$this->askForConfirmation($input, $output, null)) {
98
            return 0;
99
        }
100
101
        $concurrency = $input->getOption('concurrency');
102
        $this->writeln("Executing migrations using " . count($paths) . " processes with a concurrency of $concurrency");
103
104
        $builder = new ProcessBuilder();
105
        $executableFinder = new PhpExecutableFinder();
106
        if (false !== ($php = $executableFinder->find())) {
107
            $builder->setPrefix($php);
108
        }
109
110
        // mandatory args and options
111
        $builderArgs = $this->createChildProcessArgs($input);
112
113
        $processes = array();
114
        /** @var MigrationDefinition $migrationDefinition */
115
        foreach($paths as $path => $count) {
116
            $this->writeln("<info>Queueing processing of: $path ($count migrations)</info>", OutputInterface::VERBOSITY_VERBOSE);
117
118
            $process = $builder
119
                ->setArguments(array_merge($builderArgs, array('--path=' . $path)))
120
                ->getProcess();
121
122
            $this->writeln('<info>Command: ' . $process->getCommandLine() . '</info>', OutputInterface::VERBOSITY_VERBOSE);
123
124
            // allow long migrations processes by default
125
            $process->setTimeout(86400);
126
            $processes[] = $process;
127
        }
128
129
        $this->writeln("Starting queued processes...");
130
131
        $this->migrationsDone = array(0, 0, 0);
132
133
        $processManager = new ProcessManager();
134
        $processManager->runParallel($processes, $concurrency, 500, array($this, 'onSubProcessOutput'));
135
136
        $failed = 0;
137
        foreach ($processes as $i => $process) {
138
            if (!$process->isSuccessful()) {
139
                $output->writeln("\n<error>Subprocess $i failed! Reason: " . $process->getErrorOutput() . "</error>\n");
140
                $failed++;
141
            }
142
        }
143
144
        if ($input->getOption('clear-cache')) {
145
            $command = $this->getApplication()->find('cache:clear');
146
            $inputArray = new ArrayInput(array('command' => 'cache:clear'));
147
            $command->run($inputArray, $output);
148
        }
149
150
        $time = microtime(true) - $start;
151
152
        $this->writeln('<info>'.$this->migrationsDone[0].' migrations executed, '.$this->migrationsDone[1].' failed, '.$this->migrationsDone[2].' skipped</info>');
153
154
        // since we use subprocesses, we can not measure max memory used
155
        $this->writeln("Time taken: ".sprintf('%.2f', $time)." secs");
156
157
        return $failed;
158
    }
159
160
    protected function executeAsChild($input, $output, $toExecute, $force, $migrationService)
161
    {
162
        // @todo disable signal slots that are harmful during migrations, if any
163
164
        if ($input->getOption('separate-process')) {
165
            $builder = new ProcessBuilder();
166
            $executableFinder = new PhpExecutableFinder();
167
            if (false !== $php = $executableFinder->find()) {
168
                $builder->setPrefix($php);
169
            }
170
171
            $builderArgs = parent::createChildProcessArgs($input);
172
        }
173
174
        $failed = 0;
175
        $executed = 0;
176
        $skipped = 0;
177
        $total = count($toExecute);
178
179
        foreach ($toExecute as  $name => $migrationDefinition) {
180
            // let's skip migrations that we know are invalid - user was warned and he decided to proceed anyway
181
            if ($migrationDefinition->status == MigrationDefinition::STATUS_INVALID) {
182
                $this->writeln("<comment>Skipping migration (invalid definition?) Path: ".$migrationDefinition->path."</comment>", self::VERBOSITY_CHILD);
0 ignored issues
show
Bug introduced by
self::VERBOSITY_CHILD of type double is incompatible with the type integer expected by parameter $verbosity of Kaliop\eZMigrationBundle...grateCommand::writeln(). ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

182
                $this->writeln("<comment>Skipping migration (invalid definition?) Path: ".$migrationDefinition->path."</comment>", /** @scrutinizer ignore-type */ self::VERBOSITY_CHILD);
Loading history...
183
                $skipped++;
184
                continue;
185
            }
186
187
            if ($input->getOption('separate-process')) {
188
189
                try {
190
                    $this->executeMigrationInSeparateProcess($migrationDefinition, $migrationService, $builder, $builderArgs, false);
0 ignored issues
show
Comprehensibility Best Practice introduced by
The variable $builderArgs does not seem to be defined for all execution paths leading up to this point.
Loading history...
Comprehensibility Best Practice introduced by
The variable $builder does not seem to be defined for all execution paths leading up to this point.
Loading history...
191
192
                    $executed++;
193
                } catch (\Exception $e) {
194
                    if ($input->getOption('ignore-failures')) {
195
                        $output->writeln("\n<error>Migration failed! Reason: " . $e->getMessage() . "</error>\n");
196
                        $failed++;
197
                        continue;
198
                    }
199
                    $output->writeln("\n<error>Migration aborted! Path: " . $migrationDefinition->path . ", Reason: " . $e->getMessage() . "</error>");
200
201
                    $missed = $total - $executed - $failed - $skipped;
202
                    $this->writeln("Migrations executed: $executed, failed: $failed, skipped: $skipped, to do: $missed");
203
204
                    return 1;
205
                }
206
207
            } else {
208
209
                try {
210
                    $this->executeMigrationInProcess($migrationDefinition, $force, $migrationService, $input);
211
212
                    $executed++;
213
                } catch(\Exception $e) {
214
                    $failed++;
215
                    if ($input->getOption('ignore-failures')) {
216
                        $this->writeln("<error>Migration failed! Path: " . $migrationDefinition->path . ", Reason: " . $e->getMessage() . "</error>", self::VERBOSITY_CHILD);
217
                        continue;
218
                    }
219
220
                    $this->writeln("<error>Migration aborted! Path: " . $migrationDefinition->path . ", Reason: " . $e->getMessage() . "</error>", self::VERBOSITY_CHILD);
221
222
                    $missed = $total - $executed - $failed - $skipped;
223
                    $this->writeln("Migrations executed: $executed, failed: $failed, skipped: $skipped, to do: $missed");
224
225
                    return 1;
226
                }
227
228
            }
229
        }
230
231
        $this->writeln("Migrations executed: $executed, failed: $failed, skipped: $skipped", self::VERBOSITY_CHILD);
232
233
        // We do not return an error code > 0 if migrations fail, but only on proper fatals.
234
        // The parent will analyze the output of the child process to gather the number of executed/failed migrations anyway
235
        //return $failed;
236
    }
237
238
    public function onSubProcessOutput($type, $buffer, $process=null)
239
    {
240
        $lines = explode("\n", trim($buffer));
241
242
        foreach ($lines as $line) {
243
            if (preg_match('/Migrations executed: ([0-9]+), failed: ([0-9]+), skipped: ([0-9]+)/', $line, $matches)) {
244
                $this->migrationsDone[0] += $matches[1];
245
                $this->migrationsDone[1] += $matches[2];
246
                $this->migrationsDone[2] += $matches[3];
247
248
                // swallow these lines unless we are in verbose mode
249
                if ($this->verbosity <= Output::VERBOSITY_NORMAL) {
250
                    return;
251
                }
252
            }
253
254
            // we tag the output from the different processes
255
            if (trim($line) !== '') {
256
                echo '[' . ($process ? $process->getPid() : ''). '] ' . trim($line) . "\n";
257
            }
258
        }
259
    }
260
261
    /**
262
     * @param string $paths
263
     * @param $migrationService
264
     * @param bool $force
265
     * @param bool $isChild when not in child mode, do not waste time parsing migrations
266
     * @return MigrationDefinition[] parsed or unparsed, depending on
267
     *
268
     * @todo this does not scale well with many definitions or migrations
269
     */
270
    protected function buildMigrationsList($paths, $migrationService, $force = false, $isChild = false)
271
    {
272
        $migrationDefinitions = $migrationService->getMigrationsDefinitions($paths);
273
        $migrations = $migrationService->getMigrations();
274
275
        $this->migrationsAlreadyDone = array(Migration::STATUS_DONE => 0, Migration::STATUS_FAILED => 0, Migration::STATUS_SKIPPED => 0, Migration::STATUS_STARTED => 0);
276
277
        $allowedStatuses = array(Migration::STATUS_TODO);
278
        if ($force) {
279
            $allowedStatuses = array_merge($allowedStatuses, array(Migration::STATUS_DONE, Migration::STATUS_FAILED, Migration::STATUS_SKIPPED));
280
        }
281
282
        // filter away all migrations except 'to do' ones
283
        $toExecute = array();
284
        foreach($migrationDefinitions as $name => $migrationDefinition) {
285
            if (!isset($migrations[$name]) || (($migration = $migrations[$name]) && in_array($migration->status, $allowedStatuses))) {
286
                $toExecute[$name] = $isChild ? $migrationService->parseMigrationDefinition($migrationDefinition) : $migrationDefinition;
287
            }
288
            // save the list of non-executable migrations as well (even when using 'force')
289
            if (!$isChild && isset($migrations[$name]) && (($migration = $migrations[$name]) && $migration->status != Migration::STATUS_TODO)) {
290
                $this->migrationsAlreadyDone[$migration->status]++;
291
            }
292
        }
293
294
        // if user wants to execute 'all' migrations: look for some which are registered in the database even if not
295
        // found by the loader
296
        if (empty($paths)) {
297
            foreach ($migrations as $migration) {
298
                if (in_array($migration->status, $allowedStatuses) && !isset($toExecute[$migration->name])) {
299
                    $migrationDefinitions = $migrationService->getMigrationsDefinitions(array($migration->path));
300
                    if (count($migrationDefinitions)) {
301
                        $migrationDefinition = reset($migrationDefinitions);
302
                        $toExecute[$migration->name] = $isChild ? $migrationService->parseMigrationDefinition($migrationDefinition) : $migrationDefinition;
303
                    } else {
304
                        // q: shall we raise a warning here ?
305
                    }
306
                }
307
            }
308
        }
309
310
        ksort($toExecute);
311
312
        return $toExecute;
313
    }
314
315
    /**
316
     * We use a more compact output when there are *many* migrations
317
     * @param MigrationDefinition[] $toExecute
318
     * @param array $paths
319
     * @param InputInterface $input
320
     * @param OutputInterface $output
321
     */
322
    protected function printMigrationsList($toExecute, InputInterface $input, OutputInterface $output, $paths = array())
323
    {
324
        $output->writeln('Found ' . count($toExecute) . ' migrations in ' . count($paths) . ' directories');
325
        $output->writeln('In the same directories, migrations previously executed: ' . $this->migrationsAlreadyDone[Migration::STATUS_DONE] .
326
            ', failed: ' . $this->migrationsAlreadyDone[Migration::STATUS_FAILED] . ', skipped: '. $this->migrationsAlreadyDone[Migration::STATUS_SKIPPED]);
327
        if ($this->migrationsAlreadyDone[Migration::STATUS_STARTED]) {
328
            $output->writeln('<info>In the same directories, migrations currently executing: ' . $this->migrationsAlreadyDone[Migration::STATUS_STARTED] . '</info>');
329
        }
330
    }
331
332
    /**
333
     * @param MigrationDefinition[] $toExecute
334
     * @return array key: folder name, value: number of migrations found
335
     */
336
    protected function groupMigrationsByPath($toExecute)
337
    {
338
        $paths = array();
339
        foreach($toExecute as $name => $migrationDefinition) {
340
            $path = dirname($migrationDefinition->path);
341
            if (!isset($paths[$path])) {
342
                $paths[$path] = 1;
343
            } else {
344
                $paths[$path]++;
345
            }
346
        }
347
348
        ksort($paths);
349
350
        return $paths;
351
    }
352
353
    /**
354
     * Returns the command-line arguments needed to execute a migration in a separate subprocess (omitting 'path')
355
     * @param InputInterface $input
356
     * @return array
357
     */
358
    protected function createChildProcessArgs(InputInterface $input)
359
    {
360
        $kernel = $this->getContainer()->get('kernel');
361
362
        // mandatory args and options
363
        $builderArgs = array(
364
            $_SERVER['argv'][0], // sf console
365
            self::COMMAND_NAME, // name of sf command. Can we get it from the Application instead of hardcoding?
366
            '--env=' . $kernel-> getEnvironment(), // sf env
367
            '--child'
368
        );
369
        // sf/ez env options
370
        if (!$kernel->isDebug()) {
371
            $builderArgs[] = '--no-debug';
372
        }
373
        if ($input->getOption('siteaccess')) {
374
            $builderArgs[] = '--siteaccess=' . $input->getOption('siteaccess');
375
        }
376
        // 'optional' options
377
        // note: options 'clear-cache', 'no-interaction', 'path' we never propagate
378
        if ($input->getOption('admin-login')) {
379
            $builderArgs[] = '--admin-login=' . $input->getOption('admin-login');
380
        }
381
        if ($input->getOption('default-language')) {
382
            $builderArgs[] = '--default-language=' . $input->getOption('default-language');
383
        }
384
        if ($input->getOption('force')) {
385
            $builderArgs[] = '--force';
386
        }
387
        if ($input->getOption('ignore-failures')) {
388
            $builderArgs[] = '--ignore-failures';
389
        }
390
        if ($input->getOption('no-transactions')) {
391
            $builderArgs[] = '--no-transactions';
392
        }
393
        if ($input->getOption('separate-process')) {
394
            $builderArgs[] = '--separate-process';
395
        }
396
397
        return $builderArgs;
398
    }
399
}
400