1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* import json data into graviton |
4
|
|
|
* |
5
|
|
|
* Supports importing json data from either a single file or a complete folder of files. |
6
|
|
|
* |
7
|
|
|
* The data needs to contain frontmatter to hint where the bits and pieces should go. |
8
|
|
|
*/ |
9
|
|
|
|
10
|
|
|
namespace Graviton\ImportExport\Command; |
11
|
|
|
|
12
|
|
|
use Graviton\ImportExport\Exception\MissingTargetException; |
13
|
|
|
use Graviton\ImportExport\Exception\JsonParseException; |
14
|
|
|
use Graviton\ImportExport\Exception\UnknownFileTypeException; |
15
|
|
|
use Graviton\ImportExport\Service\HttpClient; |
16
|
|
|
use GuzzleHttp\Exception\ClientException; |
17
|
|
|
use Symfony\Component\Console\Input\InputArgument; |
18
|
|
|
use Symfony\Component\Console\Input\InputOption; |
19
|
|
|
use Symfony\Component\Console\Input\InputInterface; |
20
|
|
|
use Symfony\Component\Console\Output\OutputInterface; |
21
|
|
|
use Symfony\Component\Finder\Finder; |
22
|
|
|
use Symfony\Component\Yaml\Parser; |
23
|
|
|
use Symfony\Component\VarDumper\Cloner\VarCloner; |
24
|
|
|
use Symfony\Component\VarDumper\Dumper\CliDumper as Dumper; |
25
|
|
|
use GuzzleHttp\Promise; |
26
|
|
|
use GuzzleHttp\Exception\RequestException; |
27
|
|
|
use Symfony\Component\Finder\SplFileInfo; |
28
|
|
|
use Webuni\FrontMatter\FrontMatter; |
29
|
|
|
use Webuni\FrontMatter\Document; |
30
|
|
|
use Psr\Http\Message\ResponseInterface; |
31
|
|
|
|
32
|
|
|
/** |
33
|
|
|
* @author List of contributors <https://github.com/libgraviton/import-export/graphs/contributors> |
34
|
|
|
* @license http://opensource.org/licenses/gpl-license.php GNU Public License |
35
|
|
|
* @link http://swisscom.ch |
36
|
|
|
*/ |
37
|
|
|
class ImportCommand extends ImportCommandAbstract |
38
|
|
|
{ |
39
|
|
|
/** |
40
|
|
|
* @var HttpClient |
41
|
|
|
*/ |
42
|
|
|
private $client; |
43
|
|
|
|
44
|
|
|
/** |
45
|
|
|
* @var FrontMatter |
46
|
|
|
*/ |
47
|
|
|
private $frontMatter; |
48
|
|
|
|
49
|
|
|
/** |
50
|
|
|
* @var Parser |
51
|
|
|
*/ |
52
|
|
|
private $parser; |
53
|
|
|
|
54
|
|
|
/** |
55
|
|
|
* @var VarCloner |
56
|
|
|
*/ |
57
|
|
|
private $cloner; |
58
|
|
|
|
59
|
|
|
/** |
60
|
|
|
* @var Dumper |
61
|
|
|
*/ |
62
|
|
|
private $dumper; |
63
|
|
|
|
64
|
|
|
/** |
65
|
|
|
* Count of errors |
66
|
|
|
* @var array |
67
|
|
|
*/ |
68
|
|
|
private $errors = []; |
69
|
|
|
|
70
|
|
|
/** |
71
|
|
|
* Header basic auth |
72
|
|
|
* @var string |
73
|
|
|
*/ |
74
|
|
|
private $headerBasicAuth; |
75
|
|
|
|
76
|
|
|
/** |
77
|
|
|
* @param HttpClient $client Grv HttpClient guzzle http client |
78
|
|
|
* @param Finder $finder symfony/finder instance |
79
|
|
|
* @param FrontMatter $frontMatter frontmatter parser |
80
|
|
|
* @param Parser $parser yaml/json parser |
81
|
|
|
* @param VarCloner $cloner var cloner for dumping reponses |
82
|
|
|
* @param Dumper $dumper dumper for outputing responses |
83
|
|
|
*/ |
84
|
5 |
|
public function __construct( |
85
|
|
|
HttpClient $client, |
86
|
|
|
Finder $finder, |
87
|
|
|
FrontMatter $frontMatter, |
88
|
|
|
Parser $parser, |
89
|
|
|
VarCloner $cloner, |
90
|
|
|
Dumper $dumper |
91
|
|
|
) { |
92
|
5 |
|
parent::__construct( |
93
|
|
|
$finder |
94
|
5 |
|
); |
95
|
5 |
|
$this->client = $client; |
96
|
5 |
|
$this->frontMatter = $frontMatter; |
97
|
5 |
|
$this->parser = $parser; |
98
|
5 |
|
$this->cloner = $cloner; |
99
|
5 |
|
$this->dumper = $dumper; |
100
|
5 |
|
} |
101
|
|
|
|
102
|
|
|
/** |
103
|
|
|
* Configures the current command. |
104
|
|
|
* |
105
|
|
|
* @return void |
106
|
|
|
*/ |
107
|
5 |
|
protected function configure() |
108
|
|
|
{ |
109
|
5 |
|
$this |
110
|
5 |
|
->setName('graviton:import') |
111
|
5 |
|
->setDescription('Import files from a folder or file.') |
112
|
5 |
|
->addOption( |
113
|
5 |
|
'rewrite-host', |
114
|
5 |
|
'r', |
115
|
5 |
|
InputOption::VALUE_OPTIONAL, |
116
|
5 |
|
'Replace the value of this option with the <host> value before importing.', |
117
|
|
|
'http://localhost' |
118
|
5 |
|
) |
119
|
5 |
|
->addOption( |
120
|
5 |
|
'rewrite-to', |
121
|
5 |
|
't', |
122
|
5 |
|
InputOption::VALUE_OPTIONAL, |
123
|
5 |
|
'String to use as the replacement value for the [REWRITE-HOST] string.', |
124
|
|
|
'<host>' |
125
|
5 |
|
) |
126
|
5 |
|
->addOption( |
127
|
5 |
|
'sync-requests', |
128
|
5 |
|
's', |
129
|
5 |
|
InputOption::VALUE_NONE, |
130
|
|
|
'Send requests synchronously' |
131
|
5 |
|
) |
132
|
5 |
|
->addOption( |
133
|
5 |
|
'headers-basic-auth', |
134
|
5 |
|
'a', |
135
|
5 |
|
InputOption::VALUE_OPTIONAL, |
136
|
|
|
'Header user:password for Basic auth' |
137
|
5 |
|
) |
138
|
5 |
|
->addOption( |
139
|
5 |
|
'input-file', |
140
|
5 |
|
'i', |
141
|
5 |
|
InputOption::VALUE_REQUIRED, |
142
|
|
|
'If provided, the list of files to load will be loaded from this file, one file per line.' |
143
|
5 |
|
) |
144
|
5 |
|
->addArgument( |
145
|
5 |
|
'host', |
146
|
5 |
|
InputArgument::REQUIRED, |
147
|
|
|
'Protocol and host to load data into (ie. https://graviton.nova.scapp.io)' |
148
|
5 |
|
) |
149
|
5 |
|
->addArgument( |
150
|
5 |
|
'file', |
151
|
5 |
|
InputArgument::IS_ARRAY, |
152
|
|
|
'Directories or files to load' |
153
|
5 |
|
); |
154
|
5 |
|
} |
155
|
|
|
|
156
|
|
|
/** |
157
|
|
|
* Executes the current command. |
158
|
|
|
* |
159
|
|
|
* @param Finder $finder Finder |
160
|
|
|
* @param InputInterface $input User input on console |
161
|
|
|
* @param OutputInterface $output Output of the command |
162
|
|
|
* |
163
|
|
|
* @return void |
164
|
|
|
*/ |
165
|
5 |
|
protected function doImport(Finder $finder, InputInterface $input, OutputInterface $output) |
166
|
|
|
{ |
167
|
5 |
|
$exitCode = 0; |
168
|
5 |
|
$host = $input->getArgument('host'); |
169
|
5 |
|
$rewriteHost = $input->getOption('rewrite-host'); |
170
|
5 |
|
$rewriteTo = $input->getOption('rewrite-to'); |
171
|
5 |
|
$this->headerBasicAuth = $input->getOption('headers-basic-auth'); |
172
|
5 |
|
if ($rewriteTo === $this->getDefinition()->getOption('rewrite-to')->getDefault()) { |
173
|
5 |
|
$rewriteTo = $host; |
174
|
5 |
|
} |
175
|
5 |
|
$sync = $input->getOption('sync-requests'); |
176
|
|
|
|
177
|
5 |
|
$this->importPaths($finder, $output, $host, $rewriteHost, $rewriteTo, $sync); |
178
|
|
|
|
179
|
|
|
// Error exit |
180
|
4 |
|
if (empty($this->errors)) { |
181
|
|
|
// No errors |
182
|
3 |
|
$output->writeln("\n".'<info>No errors</info>'); |
183
|
3 |
|
} else { |
184
|
|
|
// Yes, there was errors |
185
|
1 |
|
$output->writeln("\n".'<info>There was errors: '.count($this->errors).'</info>'); |
186
|
1 |
|
foreach ($this->errors as $file => $error) { |
187
|
1 |
|
$output->writeln("<error>{$file}: {$error}</error>"); |
188
|
1 |
|
} |
189
|
1 |
|
$exitCode = 1; |
190
|
|
|
} |
191
|
4 |
|
return $exitCode; |
192
|
|
|
} |
193
|
|
|
|
194
|
|
|
/** |
195
|
|
|
* @param Finder $finder finder primmed with files to import |
196
|
|
|
* @param OutputInterface $output output interfac |
197
|
|
|
* @param string $host host to import into |
198
|
|
|
* @param string $rewriteHost string to replace with value from $rewriteTo during loading |
199
|
|
|
* @param string $rewriteTo string to replace value from $rewriteHost with during loading |
200
|
|
|
* @param boolean $sync send requests syncronously |
201
|
|
|
* |
202
|
|
|
* @return void |
203
|
|
|
* |
204
|
|
|
* @throws MissingTargetException |
205
|
|
|
*/ |
206
|
5 |
|
protected function importPaths( |
207
|
|
|
Finder $finder, |
208
|
|
|
OutputInterface $output, |
209
|
|
|
$host, |
210
|
|
|
$rewriteHost, |
211
|
|
|
$rewriteTo, |
212
|
|
|
$sync = false |
213
|
|
|
) { |
214
|
5 |
|
$promises = []; |
215
|
|
|
/** @var SplFileInfo $file */ |
216
|
5 |
|
foreach ($finder as $file) { |
217
|
5 |
|
$doc = $this->frontMatter->parse($file->getContents()); |
218
|
|
|
|
219
|
5 |
|
$output->writeln("<info>Loading data from ${file}</info>"); |
220
|
|
|
|
221
|
5 |
|
if (!array_key_exists('target', $doc->getData())) { |
222
|
1 |
|
throw new MissingTargetException('Missing target in \'' . $file . '\''); |
223
|
|
|
} |
224
|
|
|
|
225
|
4 |
|
$targetUrl = sprintf('%s%s', $host, $doc->getData()['target']); |
226
|
|
|
|
227
|
4 |
|
$promises[] = $this->importResource( |
228
|
4 |
|
$targetUrl, |
229
|
4 |
|
(string) $file, |
230
|
4 |
|
$output, |
231
|
4 |
|
$doc, |
232
|
4 |
|
$rewriteHost, |
233
|
4 |
|
$rewriteTo, |
234
|
|
|
$sync |
235
|
4 |
|
); |
236
|
4 |
|
} |
237
|
|
|
|
238
|
|
|
try { |
|
|
|
|
239
|
|
|
//Promise\unwrap($promises); |
240
|
|
|
} catch (ClientException $e) { |
|
|
|
|
241
|
|
|
// silently ignored since we already output an error when the promise fails |
242
|
|
|
} |
243
|
|
|
} |
244
|
|
|
|
245
|
|
|
/** |
246
|
|
|
* @param string $targetUrl target url to import resource into |
247
|
|
|
* @param string $file path to file being loaded |
248
|
|
|
* @param OutputInterface $output output of the command |
249
|
|
|
* @param Document $doc document to load |
250
|
|
|
* @param string $rewriteHost string to replace with value from $host during loading |
251
|
|
|
* @param string $rewriteTo string to replace value from $rewriteHost with during loading |
252
|
|
|
* @param boolean $sync send requests syncronously |
253
|
|
|
* |
254
|
|
|
* @return Promise\Promise|null |
255
|
|
|
*/ |
256
|
|
|
protected function importResource( |
257
|
|
|
$targetUrl, |
258
|
|
|
$file, |
259
|
|
|
OutputInterface $output, |
260
|
|
|
Document $doc, |
261
|
|
|
$rewriteHost, |
262
|
|
|
$rewriteTo, |
263
|
|
|
$sync = false |
264
|
|
|
) { |
265
|
4 |
|
$content = str_replace($rewriteHost, $rewriteTo, $doc->getContent()); |
266
|
4 |
|
$uploadFile = $this->validateUploadFile($doc, $file); |
267
|
|
|
|
268
|
|
|
$successFunc = function (ResponseInterface $response) use ($output) { |
269
|
3 |
|
$output->writeln( |
270
|
3 |
|
'<comment>Wrote ' . $response->getHeader('Link')[0] . '</comment>' |
271
|
3 |
|
); |
272
|
4 |
|
}; |
273
|
|
|
|
274
|
|
|
$errFunc = function (RequestException $e) use ($output, $file) { |
275
|
1 |
|
$this->errors[$file] = $e->getMessage(); |
276
|
1 |
|
$output->writeln( |
277
|
1 |
|
'<error>' . str_pad( |
278
|
1 |
|
sprintf( |
279
|
1 |
|
'Failed to write <%s> from \'%s\' with message \'%s\'', |
280
|
1 |
|
$e->getRequest()->getUri(), |
281
|
1 |
|
$file, |
282
|
1 |
|
$e->getMessage() |
283
|
1 |
|
), |
284
|
1 |
|
140, |
285
|
|
|
' ' |
286
|
1 |
|
) . '</error>' |
287
|
1 |
|
); |
288
|
1 |
|
if ($output->getVerbosity() >= OutputInterface::VERBOSITY_VERBOSE) { |
289
|
1 |
|
$this->dumper->dump( |
290
|
1 |
|
$this->cloner->cloneVar( |
291
|
1 |
|
$this->parser->parse($e->getResponse()->getBody(), false, false, true) |
292
|
1 |
|
), |
293
|
|
|
function ($line, $depth) use ($output) { |
294
|
1 |
|
if ($depth > 0) { |
295
|
1 |
|
$output->writeln( |
296
|
1 |
|
'<error>' . str_pad(str_repeat(' ', $depth) . $line, 140, ' ') . '</error>' |
297
|
1 |
|
); |
298
|
1 |
|
} |
299
|
1 |
|
} |
300
|
1 |
|
); |
301
|
1 |
|
} |
302
|
4 |
|
}; |
303
|
|
|
|
304
|
|
|
$data = [ |
305
|
4 |
|
'json' => $this->parseContent($content, $file), |
306
|
|
|
'upload' => $uploadFile |
307
|
4 |
|
]; |
308
|
4 |
|
if ($this->headerBasicAuth) { |
309
|
|
|
$data['headers'] = ['Authorization' => 'Basic '. base64_encode($this->headerBasicAuth)]; |
310
|
|
|
} |
311
|
4 |
|
$promise = $this->client->requestAsync( |
312
|
4 |
|
'PUT', |
313
|
4 |
|
$targetUrl, |
314
|
|
|
$data |
315
|
4 |
|
); |
316
|
|
|
|
317
|
|
|
// If there is a file to be uploaded, and it exists in remote, we delete it first. |
318
|
|
|
// TODO This part, $uploadFile, promise should be removed once Graviton/File service is resolved in new Story. |
319
|
4 |
|
$fileRepeatFunc = false; |
320
|
4 |
|
if ($uploadFile) { |
|
|
|
|
321
|
|
|
$fileRepeatFunc = function () use ($targetUrl, $successFunc, $errFunc, $output, $file, $data) { |
322
|
|
|
unset($this->errors[$file]); |
323
|
|
|
$output->writeln('<info>File deleting: '.$targetUrl.'</info>'); |
324
|
|
|
$deleteRequest = $this->client->requestAsync('DELETE', $targetUrl); |
325
|
|
|
$insert = function () use ($targetUrl, $successFunc, $errFunc, $output, $data) { |
326
|
|
|
$output->writeln('<info>File inserting: '.$targetUrl.'</info>'); |
327
|
|
|
$promiseInsert = $this->client->requestAsync('PUT', $targetUrl, $data); |
328
|
|
|
$promiseInsert->then($successFunc, $errFunc); |
329
|
|
|
}; |
330
|
|
|
$deleteRequest |
331
|
|
|
->then($insert, $errFunc)->wait(); |
332
|
1 |
|
}; |
333
|
1 |
|
} |
334
|
|
|
|
335
|
4 |
|
$promiseError = $fileRepeatFunc ? $fileRepeatFunc : $errFunc; |
336
|
4 |
|
if ($sync) { |
337
|
|
|
$promise->then($successFunc, $promiseError)->wait(); |
338
|
|
|
} else { |
339
|
4 |
|
$promise->then($successFunc, $promiseError); |
340
|
|
|
} |
341
|
|
|
|
342
|
|
|
|
343
|
|
|
|
344
|
4 |
|
return $promise; |
345
|
|
|
} |
346
|
|
|
|
347
|
|
|
/** |
348
|
|
|
* parse contents of a file depending on type |
349
|
|
|
* |
350
|
|
|
* @param string $content contents part of file |
351
|
|
|
* @param string $file full path to file |
352
|
|
|
* |
353
|
|
|
* @return mixed |
354
|
|
|
* @throws UnknownFileTypeException |
355
|
|
|
* @throws JsonParseException |
356
|
|
|
*/ |
357
|
|
|
protected function parseContent($content, $file) |
358
|
|
|
{ |
359
|
4 |
|
if (substr($file, -5) == '.json') { |
360
|
3 |
|
$data = json_decode($content); |
361
|
3 |
|
if (json_last_error() !== JSON_ERROR_NONE) { |
362
|
|
|
throw new JsonParseException( |
363
|
|
|
sprintf( |
364
|
|
|
'%s in %s', |
365
|
|
|
json_last_error_msg(), |
366
|
|
|
$file |
367
|
|
|
) |
368
|
|
|
); |
369
|
|
|
} |
370
|
4 |
|
} elseif (substr($file, -4) == '.yml') { |
371
|
1 |
|
$data = $this->parser->parse($content); |
372
|
1 |
|
} else { |
373
|
|
|
throw new UnknownFileTypeException($file); |
374
|
|
|
} |
375
|
|
|
|
376
|
4 |
|
return $data; |
377
|
|
|
} |
378
|
|
|
|
379
|
|
|
/** |
380
|
|
|
* Checks if file exists and return qualified fileName location |
381
|
|
|
* |
382
|
|
|
* @param Document $doc Data source for import data |
383
|
|
|
* @param string $originFile Original full filename used toimport |
384
|
|
|
* @return bool|mixed |
385
|
|
|
*/ |
386
|
|
|
private function validateUploadFile(Document $doc, $originFile) |
387
|
|
|
{ |
388
|
4 |
|
$documentData = $doc->getData(); |
389
|
|
|
|
390
|
4 |
|
if (!array_key_exists('file', $documentData)) { |
391
|
3 |
|
return false; |
392
|
|
|
} |
393
|
|
|
|
394
|
|
|
// Find file |
395
|
1 |
|
$fileName = dirname($originFile) . DIRECTORY_SEPARATOR . $documentData['file']; |
396
|
1 |
|
$fileName = str_replace('//', '/', $fileName); |
397
|
1 |
|
if (!file_exists($fileName)) { |
398
|
|
|
return false; |
399
|
|
|
} |
400
|
|
|
|
401
|
1 |
|
return $fileName; |
402
|
|
|
} |
403
|
|
|
} |
404
|
|
|
|
This check looks for
try
blocks that have no statements or where all statements have been commented out. This may be the result of changes for debugging or the code may simply be obsolete.If there is nothing in the
try
then thecatch
block can never be executed either. Thus, thesetry
statements can be removed completely.