|
1
|
|
|
<?php |
|
2
|
|
|
namespace Mathielen\ImportEngineBundle\Command; |
|
3
|
|
|
|
|
4
|
|
|
use Ddeboer\DataImport\Filter\OffsetFilter; |
|
5
|
|
|
use Mathielen\DataImport\Event\ImportItemEvent; |
|
6
|
|
|
use Mathielen\ImportEngine\Event\ImportConfigureEvent; |
|
7
|
|
|
use Mathielen\ImportEngine\Event\ImportRequestEvent; |
|
8
|
|
|
use Mathielen\ImportEngine\Exception\InvalidConfigurationException; |
|
9
|
|
|
use Mathielen\ImportEngine\Import\ImportBuilder; |
|
10
|
|
|
use Mathielen\ImportEngine\Import\Run\ImportRunner; |
|
11
|
|
|
use Mathielen\ImportEngine\ValueObject\ImportRequest; |
|
12
|
|
|
use Mathielen\ImportEngine\ValueObject\ImportRun; |
|
13
|
|
|
use Mathielen\ImportEngineBundle\Utils; |
|
14
|
|
|
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; |
|
15
|
|
|
use Symfony\Component\Console\Helper\ProgressBar; |
|
16
|
|
|
use Symfony\Component\Console\Helper\Table; |
|
17
|
|
|
use Symfony\Component\Console\Helper\TableSeparator; |
|
18
|
|
|
use Symfony\Component\Console\Input\InputArgument; |
|
19
|
|
|
use Symfony\Component\Console\Input\InputInterface; |
|
20
|
|
|
use Symfony\Component\Console\Input\InputOption; |
|
21
|
|
|
use Symfony\Component\Console\Output\OutputInterface; |
|
22
|
|
|
use Symfony\Component\Validator\ConstraintViolation; |
|
23
|
|
|
|
|
24
|
|
|
class ImportCommand extends ContainerAwareCommand |
|
25
|
|
|
{ |
|
26
|
|
|
|
|
27
|
|
|
const MAX_VIOLATION_ERRORS = 10; |
|
28
|
|
|
|
|
29
|
4 |
|
protected function configure() |
|
30
|
|
|
{ |
|
31
|
4 |
|
$this->setName('importengine:import') |
|
32
|
4 |
|
->setDescription('Imports data with a definied importer') |
|
33
|
4 |
|
->addArgument('source_id', InputArgument::REQUIRED, "id of source. Different StorageProviders need different id data.\n- upload, directory: \"<path/to/file>\"\n- doctrine: \"<id of query>\"\n- service: \"<service>.<method>[?arguments_like_url_query]\"") |
|
34
|
4 |
|
->addArgument('source_provider', InputArgument::OPTIONAL, 'id of source provider', 'default') |
|
35
|
4 |
|
->addOption('importer', 'i', InputOption::VALUE_OPTIONAL, 'id/name of importer') |
|
36
|
4 |
|
->addOption('context', 'c', InputOption::VALUE_OPTIONAL, 'Supply optional context information to import. Supply key-value data in query style: key=value&otherkey=othervalue&...') |
|
37
|
4 |
|
->addOption('limit', 'l', InputOption::VALUE_OPTIONAL, 'Limit imported rows') |
|
38
|
4 |
|
->addOption('dryrun', 'd', InputOption::VALUE_NONE, 'Do not import - Validation only') |
|
39
|
|
|
; |
|
40
|
4 |
|
} |
|
41
|
|
|
|
|
42
|
4 |
|
protected function validateInput(InputInterface $input) |
|
|
|
|
|
|
43
|
|
|
{ |
|
44
|
4 |
|
if (!$this->getContainer()->has('mathielen_importengine.import.builder') || |
|
45
|
4 |
|
!$this->getContainer()->has('mathielen_importengine.import.runner')) { |
|
46
|
|
|
throw new InvalidConfigurationException("No importengine services have been found. Did you register the bundle in AppKernel and configured at least one importer in config?"); |
|
47
|
|
|
} |
|
48
|
4 |
|
} |
|
49
|
|
|
|
|
50
|
4 |
|
protected function execute(InputInterface $input, OutputInterface $output) |
|
51
|
|
|
{ |
|
52
|
4 |
|
$this->validateInput($input); |
|
53
|
|
|
|
|
54
|
4 |
|
$importerId = $input->getOption('importer'); |
|
55
|
4 |
|
$sourceProviderId = $input->getArgument('source_provider'); |
|
56
|
4 |
|
$sourceId = $input->getArgument('source_id'); |
|
57
|
4 |
|
$isDryrun = $input->getOption('dryrun'); |
|
58
|
4 |
|
if ($context = $input->getOption('context')) { |
|
59
|
|
|
//parse key=value&key=value string to array |
|
60
|
|
|
if (strpos($context, '=') !== false) { |
|
61
|
|
|
parse_str($input->getOption('context'), $context); |
|
62
|
|
|
} |
|
63
|
|
|
} |
|
64
|
4 |
|
$limit = $input->getOption('limit'); |
|
65
|
|
|
|
|
66
|
4 |
|
$this->import($output, $importerId, $sourceProviderId, $sourceId, $context, $limit, $isDryrun); |
|
67
|
4 |
|
} |
|
68
|
|
|
|
|
69
|
4 |
|
protected function import(OutputInterface $output, $importerId, $sourceProviderId, $sourceId, $context=null, $limit=null, $isDryrun=false) |
|
70
|
|
|
{ |
|
71
|
4 |
|
$output->writeln("Commencing ".($isDryrun?'<comment>dry-run</comment> ':'')."import using importer ".(empty($importerId)?'<comment>unknown</comment>':"<info>$importerId</info>")." with source provider <info>$sourceProviderId</info> and source id <info>$sourceId</info>"); |
|
72
|
|
|
|
|
73
|
4 |
|
$sourceId = Utils::parseSourceId($sourceId); |
|
74
|
4 |
|
$progress = new ProgressBar($output); |
|
75
|
|
|
|
|
76
|
|
|
//set limit |
|
77
|
4 |
|
if ($limit) { |
|
78
|
|
|
$this->getContainer()->get('event_dispatcher')->addListener(ImportConfigureEvent::AFTER_BUILD, function (ImportConfigureEvent $event) use ($limit) { |
|
79
|
|
|
$event->getImport()->importer()->filters()->add(new OffsetFilter(0, $limit)); |
|
80
|
1 |
|
}); |
|
81
|
|
|
} |
|
82
|
|
|
|
|
83
|
|
|
//show discovered importer id |
|
84
|
4 |
|
if (empty($importerId)) { |
|
85
|
|
|
$this->getContainer()->get('event_dispatcher')->addListener(ImportRequestEvent::DISCOVERED, function (ImportRequestEvent $event) use ($output) { |
|
86
|
|
|
$importerId = $event->getImportRequest()->getImporterId(); |
|
87
|
|
|
$output->writeln("Importer discovered: <info>$importerId</info>"); |
|
88
|
2 |
|
}); |
|
89
|
|
|
} |
|
90
|
|
|
|
|
91
|
|
|
/** @var ImportBuilder $importBuilder */ |
|
92
|
4 |
|
$importBuilder = $this->getContainer()->get('mathielen_importengine.import.builder'); |
|
93
|
|
|
|
|
94
|
4 |
|
$importRequest = new ImportRequest($sourceId, $sourceProviderId, $importerId, Utils::whoAmI().'@CLI'); |
|
95
|
|
|
|
|
96
|
4 |
|
$import = $importBuilder->buildFromRequest($importRequest); |
|
97
|
|
|
|
|
98
|
|
|
//apply context info from commandline |
|
99
|
4 |
|
$importRun = $import->getRun(); |
|
100
|
4 |
|
$importRun->setContext($context); |
|
101
|
|
|
|
|
102
|
|
|
//status callback |
|
103
|
4 |
|
$this->getContainer()->get('event_dispatcher')->addListener(ImportItemEvent::AFTER_READ, function (ImportItemEvent $event) use ($output, &$progress) { |
|
104
|
|
|
/** @var ImportRun $importRun */ |
|
105
|
|
|
$importRun = $event->getContext()->getRun(); |
|
106
|
|
|
$stats = $importRun->getStatistics(); |
|
107
|
|
|
$processed = isset($stats['processed'])?$stats['processed']:0; |
|
108
|
|
|
$max = $importRun->getInfo()['count']; |
|
109
|
|
|
|
|
110
|
|
|
if ($progress->getMaxSteps() != $max) { |
|
111
|
|
|
$progress = new ProgressBar($output, $max); |
|
112
|
|
|
$progress->start(); |
|
113
|
|
|
} |
|
114
|
|
|
|
|
115
|
|
|
$progress->setProgress($processed); |
|
116
|
4 |
|
}); |
|
117
|
|
|
|
|
118
|
|
|
/** @var ImportRunner $importRunner */ |
|
119
|
4 |
|
$importRunner = $this->getContainer()->get('mathielen_importengine.import.runner'); |
|
120
|
4 |
|
if ($isDryrun) { |
|
121
|
1 |
|
$importRunner->dryRun($import); |
|
122
|
|
|
} else { |
|
123
|
3 |
|
$importRunner->run($import); |
|
124
|
|
|
} |
|
125
|
|
|
|
|
126
|
4 |
|
$progress->finish(); |
|
127
|
4 |
|
$output->writeln(''); |
|
128
|
4 |
|
$output->writeln("<info>Import done</info>"); |
|
129
|
4 |
|
$output->writeln(''); |
|
130
|
|
|
|
|
131
|
4 |
|
$this->writeStatistics($importRun->getStatistics(), new Table($output)); |
|
132
|
|
|
|
|
133
|
4 |
|
$this->writeValidationViolations( |
|
134
|
|
|
$import |
|
135
|
4 |
|
->importer() |
|
136
|
4 |
|
->validation() |
|
137
|
4 |
|
->getViolations(), |
|
138
|
4 |
|
new Table($output)); |
|
139
|
|
|
|
|
140
|
4 |
|
$output->writeln(''); |
|
141
|
4 |
|
} |
|
142
|
|
|
|
|
143
|
4 |
|
protected function writeValidationViolations(array $violations, Table $table) |
|
144
|
|
|
{ |
|
145
|
4 |
|
if (empty($violations)) { |
|
146
|
4 |
|
return; |
|
147
|
|
|
} |
|
148
|
|
|
$violations = $violations['source'] + $violations['target']; |
|
149
|
|
|
|
|
150
|
|
|
$table |
|
151
|
|
|
->setHeaders(array('Constraint', 'Occurrences (lines)')) |
|
152
|
|
|
; |
|
153
|
|
|
|
|
154
|
|
|
$tree = []; |
|
155
|
|
|
foreach ($violations as $line=>$validations) { |
|
156
|
|
|
/** @var ConstraintViolation $validation */ |
|
157
|
|
|
foreach ($validations as $validation) { |
|
158
|
|
|
$key = $validation->__toString(); |
|
159
|
|
|
if (!isset($tree[$key])) { |
|
160
|
|
|
$tree[$key] = []; |
|
161
|
|
|
} |
|
162
|
|
|
$tree[$key][] = $line; |
|
163
|
|
|
} |
|
164
|
|
|
} |
|
165
|
|
|
|
|
166
|
|
|
$i = 0; |
|
167
|
|
|
foreach ($tree as $violation=>$lines) { |
|
168
|
|
|
$table->addRow([$violation, join(', ', Utils::numbersToRangeText($lines))]); |
|
169
|
|
|
++$i; |
|
170
|
|
|
|
|
171
|
|
|
if ($i === self::MAX_VIOLATION_ERRORS) { |
|
172
|
|
|
$table->addRow(new TableSeparator()); |
|
173
|
|
|
$table->addRow(array(null, 'There are more errors...')); |
|
174
|
|
|
|
|
175
|
|
|
break; |
|
176
|
|
|
} |
|
177
|
|
|
} |
|
178
|
|
|
|
|
179
|
|
|
if ($i > 0) { |
|
180
|
|
|
$table->render(); |
|
181
|
|
|
} |
|
182
|
|
|
} |
|
183
|
|
|
|
|
184
|
4 |
|
protected function writeStatistics(array $statistics, Table $table) |
|
185
|
|
|
{ |
|
186
|
4 |
|
$rows = []; |
|
187
|
4 |
|
foreach ($statistics as $k=>$v) { |
|
188
|
|
|
$rows[] = [$k, $v]; |
|
189
|
|
|
} |
|
190
|
|
|
|
|
191
|
|
|
$table |
|
192
|
4 |
|
->setHeaders(array('Statistics')) |
|
193
|
4 |
|
->setRows($rows) |
|
194
|
|
|
; |
|
195
|
4 |
|
$table->render(); |
|
196
|
4 |
|
} |
|
197
|
|
|
|
|
198
|
|
|
} |
|
199
|
|
|
|
mathielen /
ImportEngineBundle
This check looks from parameters that have been defined for a function or method, but which are not used in the method body.