1
|
|
|
<?php |
2
|
|
|
declare(strict_types=1); |
3
|
|
|
|
4
|
|
|
/** |
5
|
|
|
* BEdita, API-first content management framework |
6
|
|
|
* Copyright 2023 Atlas Srl, Chialab Srl |
7
|
|
|
* |
8
|
|
|
* This file is part of BEdita: you can redistribute it and/or modify |
9
|
|
|
* it under the terms of the GNU Lesser General Public License as published |
10
|
|
|
* by the Free Software Foundation, either version 3 of the License, or |
11
|
|
|
* (at your option) any later version. |
12
|
|
|
* |
13
|
|
|
* See LICENSE.LGPL or <http://gnu.org/licenses/lgpl-3.0.html> for more details. |
14
|
|
|
*/ |
15
|
|
|
|
16
|
|
|
namespace BEdita\ImportTools\Utility; |
17
|
|
|
|
18
|
|
|
use BEdita\Core\Model\Action\SaveEntityAction; |
19
|
|
|
use BEdita\Core\Model\Action\SetRelatedObjectsAction; |
20
|
|
|
use BEdita\Core\Model\Entity\ObjectEntity; |
21
|
|
|
use BEdita\Core\Model\Entity\Translation; |
22
|
|
|
use BEdita\Core\Model\Table\ObjectsTable; |
23
|
|
|
use BEdita\Core\Model\Table\TranslationsTable; |
24
|
|
|
use Cake\Database\Expression\FunctionExpression; |
25
|
|
|
use Cake\Database\Expression\QueryExpression; |
26
|
|
|
use Cake\Datasource\EntityInterface; |
27
|
|
|
use Cake\Http\Exception\BadRequestException; |
28
|
|
|
use Cake\Log\LogTrait; |
29
|
|
|
use Cake\ORM\Locator\LocatorAwareTrait; |
30
|
|
|
use Cake\ORM\Query; |
31
|
|
|
use Cake\ORM\Table; |
32
|
|
|
use Cake\Utility\Hash; |
33
|
|
|
use DOMDocument; |
34
|
|
|
use DOMXPath; |
35
|
|
|
|
36
|
|
|
/** |
37
|
|
|
* Import utility |
38
|
|
|
* |
39
|
|
|
* This class provides functions to import data from csv files into BEdita. |
40
|
|
|
* |
41
|
|
|
* Public methods are: |
42
|
|
|
* |
43
|
|
|
* - `saveObjects`: read data from csv and save objects |
44
|
|
|
* - `saveObject`: save a single object |
45
|
|
|
* - `saveTranslations`: read data from csv and save translations |
46
|
|
|
* - `saveTranslation`: save a single translation |
47
|
|
|
* - `translatedFields`: get translated fields for a given object |
48
|
|
|
* |
49
|
|
|
* Usage example: |
50
|
|
|
* ```php |
51
|
|
|
* use BEdita\ImportTools\Utility\Import; |
52
|
|
|
* |
53
|
|
|
* class MyImporter |
54
|
|
|
* { |
55
|
|
|
* public function import(string $filename, string $type, ?string $parent, ?bool $dryrun): void |
56
|
|
|
* { |
57
|
|
|
* $import = new Import($filename, $type, $parent, $dryrun); |
58
|
|
|
* $import->saveObjects(); |
59
|
|
|
* } |
60
|
|
|
* } |
61
|
|
|
* ``` |
62
|
|
|
*/ |
63
|
|
|
class Import |
64
|
|
|
{ |
65
|
|
|
use LocatorAwareTrait; |
66
|
|
|
use LogTrait; |
67
|
|
|
use ReadTrait; |
68
|
|
|
use TreeTrait; |
69
|
|
|
|
70
|
|
|
/** |
71
|
|
|
* @inheritDoc |
72
|
|
|
*/ |
73
|
|
|
protected $_defaultConfig = [ |
74
|
|
|
'defaults' => [ |
75
|
|
|
'status' => 'on', |
76
|
|
|
], |
77
|
|
|
'csv' => [ |
78
|
|
|
'delimiter' => ',', |
79
|
|
|
'enclosure' => '"', |
80
|
|
|
'escape' => '"', |
81
|
|
|
], |
82
|
|
|
]; |
83
|
|
|
|
84
|
|
|
/** |
85
|
|
|
* Dry run mode flag |
86
|
|
|
* |
87
|
|
|
* @var bool |
88
|
|
|
*/ |
89
|
|
|
public bool $dryrun = false; |
90
|
|
|
|
91
|
|
|
/** |
92
|
|
|
* Full filename path |
93
|
|
|
* |
94
|
|
|
* @var string|null |
95
|
|
|
*/ |
96
|
|
|
public ?string $filename = ''; |
97
|
|
|
|
98
|
|
|
/** |
99
|
|
|
* Parent uname or ID |
100
|
|
|
* |
101
|
|
|
* @var string|null |
102
|
|
|
*/ |
103
|
|
|
public ?string $parent = ''; |
104
|
|
|
|
105
|
|
|
/** |
106
|
|
|
* Number of processed entities |
107
|
|
|
* |
108
|
|
|
* @var int |
109
|
|
|
*/ |
110
|
|
|
public int $processed = 0; |
111
|
|
|
|
112
|
|
|
/** |
113
|
|
|
* Number of saved entities |
114
|
|
|
* |
115
|
|
|
* @var int |
116
|
|
|
*/ |
117
|
|
|
public int $saved = 0; |
118
|
|
|
|
119
|
|
|
/** |
120
|
|
|
* Number of errors |
121
|
|
|
* |
122
|
|
|
* @var int |
123
|
|
|
*/ |
124
|
|
|
public int $errors = 0; |
125
|
|
|
|
126
|
|
|
/** |
127
|
|
|
* Errors details |
128
|
|
|
* |
129
|
|
|
* @var array |
130
|
|
|
*/ |
131
|
|
|
public array $errorsDetails = []; |
132
|
|
|
|
133
|
|
|
/** |
134
|
|
|
* Number of skipped |
135
|
|
|
* |
136
|
|
|
* @var int |
137
|
|
|
*/ |
138
|
|
|
public int $skipped = 0; |
139
|
|
|
|
140
|
|
|
/** |
141
|
|
|
* Entity type |
142
|
|
|
* |
143
|
|
|
* @var string |
144
|
|
|
*/ |
145
|
|
|
public string $type = ''; |
146
|
|
|
|
147
|
|
|
/** |
148
|
|
|
* Source type |
149
|
|
|
* |
150
|
|
|
* @var string |
151
|
|
|
*/ |
152
|
|
|
public string $sourceType = 'csv'; |
153
|
|
|
|
154
|
|
|
/** |
155
|
|
|
* Source mapping |
156
|
|
|
* |
157
|
|
|
* @var array |
158
|
|
|
*/ |
159
|
|
|
public array $sourceMapping = []; |
160
|
|
|
|
161
|
|
|
/** |
162
|
|
|
* Objects table |
163
|
|
|
* |
164
|
|
|
* @var \BEdita\Core\Model\Table\ObjectsTable |
165
|
|
|
*/ |
166
|
|
|
protected ObjectsTable $objectsTable; |
167
|
|
|
|
168
|
|
|
/** |
169
|
|
|
* Type table |
170
|
|
|
* |
171
|
|
|
* @var \BEdita\Core\Model\Table\ObjectsTable |
172
|
|
|
*/ |
173
|
|
|
protected ObjectsTable $typeTable; |
174
|
|
|
|
175
|
|
|
/** |
176
|
|
|
* Translations table |
177
|
|
|
* |
178
|
|
|
* @var \BEdita\Core\Model\Table\TranslationsTable |
179
|
|
|
*/ |
180
|
|
|
protected TranslationsTable $translationsTable; |
181
|
|
|
|
182
|
|
|
/** |
183
|
|
|
* Assoc flag, for csv import |
184
|
|
|
* |
185
|
|
|
* @var bool |
186
|
|
|
*/ |
187
|
|
|
protected bool $assoc = true; |
188
|
|
|
|
189
|
|
|
/** |
190
|
|
|
* Element name, for xml import |
191
|
|
|
* |
192
|
|
|
* @var string |
193
|
|
|
*/ |
194
|
|
|
protected string $element = 'post'; |
195
|
|
|
|
196
|
|
|
/** |
197
|
|
|
* Constructor |
198
|
|
|
* |
199
|
|
|
* @param string|null $filename Full filename path |
200
|
|
|
* @param string|null $type Entity type |
201
|
|
|
* @param string|null $parent Parent uname or ID |
202
|
|
|
* @param bool|null $dryrun Dry run mode flag |
203
|
|
|
* @param array|null $options Options |
204
|
|
|
* @return void |
205
|
|
|
*/ |
206
|
|
|
public function __construct( |
207
|
|
|
?string $filename = null, |
208
|
|
|
?string $type = 'objects', |
209
|
|
|
?string $parent = null, |
210
|
|
|
?bool $dryrun = false, |
211
|
|
|
?array $options = ['mapping' => [], 'type' => 'csv', 'assoc' => true, 'element' => 'post'] |
212
|
|
|
) { |
213
|
|
|
$this->filename = $filename; |
214
|
|
|
$this->type = $type; |
215
|
|
|
$this->parent = $parent; |
216
|
|
|
$this->dryrun = $dryrun; |
217
|
|
|
$this->sourceMapping = Hash::get($options, 'mapping', []); |
218
|
|
|
$this->sourceType = Hash::get($options, 'type', 'csv'); |
219
|
|
|
$this->assoc = Hash::get($options, 'assoc', true); |
220
|
|
|
$this->element = Hash::get($options, 'element', 'post'); |
221
|
|
|
$this->processed = 0; |
222
|
|
|
$this->saved = 0; |
223
|
|
|
$this->errors = 0; |
224
|
|
|
$this->skipped = 0; |
225
|
|
|
$this->errorsDetails = []; |
226
|
|
|
/** @var \BEdita\Core\Model\Table\ObjectsTable $objectsTable */ |
227
|
|
|
$objectsTable = $this->fetchTable('objects'); |
228
|
|
|
$this->objectsTable = $objectsTable; |
229
|
|
|
$typesTable = $this->fetchTable($this->type); |
230
|
|
|
$this->typeTable = $typesTable instanceof ObjectsTable ? $typesTable : $objectsTable; |
231
|
|
|
/** @var \BEdita\Core\Model\Table\TranslationsTable $translationsTable */ |
232
|
|
|
$translationsTable = $this->fetchTable('translations'); |
233
|
|
|
$this->translationsTable = $translationsTable; |
234
|
|
|
} |
235
|
|
|
|
236
|
|
|
/** |
237
|
|
|
* Save media |
238
|
|
|
* |
239
|
|
|
* @param \Cake\ORM\Table $mediaTable Media table |
240
|
|
|
* @param array $mediaData Media data |
241
|
|
|
* @param array $streamData Stream data |
242
|
|
|
* @return \Cake\Datasource\EntityInterface|bool |
243
|
|
|
*/ |
244
|
|
|
public function saveMedia($mediaTable, array $mediaData, array $streamData): EntityInterface|bool |
245
|
|
|
{ |
246
|
|
|
// create media |
247
|
|
|
$media = $mediaTable->newEntity($mediaData); |
248
|
|
|
if ($this->dryrun === true) { |
249
|
|
|
$this->skipped++; |
250
|
|
|
|
251
|
|
|
return $media; |
252
|
|
|
} |
253
|
|
|
// create media |
254
|
|
|
$action = new SaveEntityAction(['table' => $mediaTable]); |
255
|
|
|
$entity = $media; |
256
|
|
|
$data = $mediaData; |
257
|
|
|
$entity = $action(compact('entity', 'data')); |
258
|
|
|
$id = $entity->id; |
259
|
|
|
|
260
|
|
|
// create stream and attach it to the media |
261
|
|
|
$streamsTable = $this->fetchTable('Streams'); |
262
|
|
|
$entity = $streamsTable->newEmptyEntity(); |
263
|
|
|
$action = new SaveEntityAction(['table' => $streamsTable]); |
264
|
|
|
$data = $streamData; |
265
|
|
|
$entity->set('object_id', $id); |
266
|
|
|
|
267
|
|
|
return $action(compact('entity', 'data')); |
268
|
|
|
} |
269
|
|
|
|
270
|
|
|
/** |
271
|
|
|
* Save objects |
272
|
|
|
* |
273
|
|
|
* @return void |
274
|
|
|
*/ |
275
|
|
|
public function saveObjects(): void |
276
|
|
|
{ |
277
|
|
|
foreach ($this->readItem($this->sourceType, $this->filename, $this->assoc, $this->element) as $obj) { |
278
|
|
|
try { |
279
|
|
|
$data = $this->transform($obj, $this->sourceMapping); |
280
|
|
|
$this->saveObject($data); |
281
|
|
|
} catch (\Exception $e) { |
282
|
|
|
$this->errorsDetails[] = $e->getMessage(); |
283
|
|
|
$this->errors++; |
284
|
|
|
} finally { |
285
|
|
|
$this->processed++; |
286
|
|
|
} |
287
|
|
|
} |
288
|
|
|
} |
289
|
|
|
|
290
|
|
|
/** |
291
|
|
|
* Save object |
292
|
|
|
* |
293
|
|
|
* @param array $obj Object data |
294
|
|
|
* @return \BEdita\Core\Model\Entity\ObjectEntity |
295
|
|
|
*/ |
296
|
|
|
public function saveObject(array $obj): ObjectEntity |
297
|
|
|
{ |
298
|
|
|
$entity = $this->typeTable->newEmptyEntity(); |
299
|
|
|
if (!empty($obj['uname']) || !empty($obj['id'])) { |
300
|
|
|
$uname = (string)Hash::get($obj, 'uname'); |
301
|
|
|
$identifier = empty($uname) ? 'id' : 'uname'; |
302
|
|
|
$conditions = [$identifier => (string)Hash::get($obj, $identifier)]; |
303
|
|
|
if ($this->objectsTable->exists($conditions)) { |
304
|
|
|
/** @var \BEdita\Core\Model\Entity\ObjectEntity $o */ |
305
|
|
|
$o = $this->objectsTable->find()->where($conditions)->firstOrFail(); |
306
|
|
|
if ($o->type !== $this->type) { |
307
|
|
|
throw new BadRequestException( |
308
|
|
|
sprintf( |
309
|
|
|
'Object "%s" already present with another type "%s"', |
310
|
|
|
$conditions[$identifier], |
311
|
|
|
$o->type |
312
|
|
|
) |
313
|
|
|
); |
314
|
|
|
} |
315
|
|
|
$entity = $o->getTable()->find('type', [$this->type])->where($conditions)->firstOrFail(); |
316
|
|
|
} |
317
|
|
|
} |
318
|
|
|
$entity = $this->typeTable->patchEntity($entity, $obj); |
319
|
|
|
$entity->set('type', $this->type); |
320
|
|
|
if ($this->dryrun === true) { |
321
|
|
|
$this->skipped++; |
322
|
|
|
|
323
|
|
|
return $entity; |
324
|
|
|
} |
325
|
|
|
$this->typeTable->saveOrFail($entity); |
326
|
|
|
if (!empty($this->parent)) { |
327
|
|
|
$this->setParent($entity, $this->parent); |
328
|
|
|
} |
329
|
|
|
$this->saved++; |
330
|
|
|
|
331
|
|
|
return $entity; |
332
|
|
|
} |
333
|
|
|
|
334
|
|
|
/** |
335
|
|
|
* Set related objects to an entity by relation |
336
|
|
|
* |
337
|
|
|
* @param string $relation Relation name |
338
|
|
|
* @param \BEdita\Core\Model\Entity\ObjectEntity $entity Entity |
339
|
|
|
* @param array $relatedEntities Related entities |
340
|
|
|
* @return array|int|false |
341
|
|
|
*/ |
342
|
|
|
public function setRelated(string $relation, ObjectEntity $entity, array $relatedEntities): array|int|false |
343
|
|
|
{ |
344
|
|
|
if (empty($relatedEntities)) { |
345
|
|
|
return false; |
346
|
|
|
} |
347
|
|
|
$association = $entity->getTable()->associations()->getByProperty($relation); |
348
|
|
|
$action = new SetRelatedObjectsAction(compact('association')); |
349
|
|
|
|
350
|
|
|
return $action(['entity' => $entity, 'relatedEntities' => $relatedEntities]); |
351
|
|
|
} |
352
|
|
|
|
353
|
|
|
/** |
354
|
|
|
* Save translations |
355
|
|
|
* |
356
|
|
|
* @return void |
357
|
|
|
*/ |
358
|
|
|
public function saveTranslations(): void |
359
|
|
|
{ |
360
|
|
|
foreach ($this->readItem($this->sourceType, $this->filename, $this->assoc, $this->element) as $translation) { |
361
|
|
|
try { |
362
|
|
|
$this->saveTranslation($translation); |
363
|
|
|
} catch (\Exception $e) { |
364
|
|
|
$this->errorsDetails[] = $e->getMessage(); |
365
|
|
|
$this->errors++; |
366
|
|
|
} finally { |
367
|
|
|
$this->processed++; |
368
|
|
|
} |
369
|
|
|
} |
370
|
|
|
} |
371
|
|
|
|
372
|
|
|
/** |
373
|
|
|
* Save translation |
374
|
|
|
* |
375
|
|
|
* @param array $data Translation data |
376
|
|
|
* @return \BEdita\Core\Model\Entity\Translation |
377
|
|
|
* @throws \Cake\Http\Exception\BadRequestException |
378
|
|
|
*/ |
379
|
|
|
public function saveTranslation(array $data): Translation |
380
|
|
|
{ |
381
|
|
|
$uname = (string)Hash::get($data, 'object_uname'); |
382
|
|
|
if (!$this->objectsTable->exists(compact('uname'))) { |
383
|
|
|
throw new BadRequestException(sprintf('Object "%s" not found', $uname)); |
384
|
|
|
} |
385
|
|
|
/** @var \BEdita\Core\Model\Entity\ObjectEntity $o */ |
386
|
|
|
$o = $this->objectsTable->find()->where(compact('uname'))->firstOrFail(); |
387
|
|
|
$objectId = $o->id; |
388
|
|
|
/** @var \BEdita\Core\Model\Entity\Translation $entity */ |
389
|
|
|
$entity = $this->translationsTable->find() |
390
|
|
|
->where([ |
391
|
|
|
'object_id' => $objectId, |
392
|
|
|
'lang' => $data['lang'], |
393
|
|
|
]) |
394
|
|
|
->first(); |
395
|
|
|
$translation = [ |
396
|
|
|
'object_id' => $objectId, |
397
|
|
|
]; |
398
|
|
|
if ($entity != null) { |
399
|
|
|
$entity = $this->translationsTable->patchEntity($entity, $translation); |
400
|
|
|
} else { |
401
|
|
|
$entity = $this->translationsTable->newEntity($translation); |
402
|
|
|
} |
403
|
|
|
$entity->set('translated_fields', $this->translatedFields($data)); |
404
|
|
|
$entity->set('status', $this->getConfig('defaults')['status']); |
405
|
|
|
$entity->set('lang', $data['lang']); |
406
|
|
|
if ($this->dryrun === true) { |
407
|
|
|
$this->skipped++; |
408
|
|
|
|
409
|
|
|
return $entity; |
410
|
|
|
} |
411
|
|
|
$this->translationsTable->saveOrFail($entity); |
412
|
|
|
$this->saved++; |
413
|
|
|
|
414
|
|
|
return $entity; |
415
|
|
|
} |
416
|
|
|
|
417
|
|
|
/** |
418
|
|
|
* Transform data into BEdita object data |
419
|
|
|
* |
420
|
|
|
* @param array $obj The source data |
421
|
|
|
* @param array $mapping The mapping |
422
|
|
|
* @return array |
423
|
|
|
*/ |
424
|
|
|
public function transform(array $obj, array $mapping): array |
425
|
|
|
{ |
426
|
|
|
if (empty($mapping)) { |
427
|
|
|
return $obj; |
428
|
|
|
} |
429
|
|
|
$data = []; |
430
|
|
|
foreach ($mapping as $key => $value) { |
431
|
|
|
if (!array_key_exists($key, $obj)) { |
432
|
|
|
continue; |
433
|
|
|
} |
434
|
|
|
$data = Hash::insert($data, $value, Hash::get($obj, $key)); |
435
|
|
|
} |
436
|
|
|
|
437
|
|
|
return $data; |
438
|
|
|
} |
439
|
|
|
|
440
|
|
|
/** |
441
|
|
|
* Get translated fields |
442
|
|
|
* |
443
|
|
|
* @param array $source Source data |
444
|
|
|
* @return array |
445
|
|
|
*/ |
446
|
|
|
public function translatedFields(array $source): array |
447
|
|
|
{ |
448
|
|
|
$fields = (string)Hash::get($source, 'translated_fields'); |
449
|
|
|
if (!empty($fields)) { |
450
|
|
|
return json_decode($fields, true); |
451
|
|
|
} |
452
|
|
|
$fields = []; |
453
|
|
|
foreach ($source as $key => $value) { |
454
|
|
|
if (in_array($key, ['id', 'object_uname', 'lang'])) { |
455
|
|
|
continue; |
456
|
|
|
} |
457
|
|
|
$subkey = strpos($key, 'translation_') === 0 ? substr($key, 12) : $key; |
458
|
|
|
$fields[$subkey] = $value; |
459
|
|
|
} |
460
|
|
|
|
461
|
|
|
return $fields; |
462
|
|
|
} |
463
|
|
|
|
464
|
|
|
/** |
465
|
|
|
* Find object by key and identifier. |
466
|
|
|
* |
467
|
|
|
* @param \Cake\ORM\Table $table Table instance. |
468
|
|
|
* @param string $extraKey Extra key. |
469
|
|
|
* @param string $extraValue Extra value. |
470
|
|
|
* @return \Cake\ORM\Query|null |
471
|
|
|
* @codeCoverageIgnore as JSON_UNQUOTE and JSON_EXTRACT are not available for sqlite |
472
|
|
|
*/ |
473
|
|
|
public function findImported(Table $table, string $extraKey, string $extraValue): ?Query |
474
|
|
|
{ |
475
|
|
|
return $table->find('available')->where(function (QueryExpression $exp) use ($table, $extraKey, $extraValue): QueryExpression { |
476
|
|
|
return $exp->and([ |
477
|
|
|
$exp->isNotNull($table->aliasField('extra')), |
478
|
|
|
$exp->eq( |
479
|
|
|
new FunctionExpression( |
480
|
|
|
'JSON_UNQUOTE', |
481
|
|
|
[ |
482
|
|
|
new FunctionExpression( |
483
|
|
|
'JSON_EXTRACT', |
484
|
|
|
['extra' => 'identifier', sprintf('$.%s', $extraKey)] |
485
|
|
|
), |
486
|
|
|
] |
487
|
|
|
), |
488
|
|
|
new FunctionExpression('JSON_UNQUOTE', [json_encode($extraValue)]) |
489
|
|
|
), |
490
|
|
|
]); |
491
|
|
|
}); |
492
|
|
|
} |
493
|
|
|
|
494
|
|
|
/** |
495
|
|
|
* Clean HTML from attributes, preserve some (using xpath expression) |
496
|
|
|
* |
497
|
|
|
* @param string $html HTML content |
498
|
|
|
* @param string $expression XPath expression |
499
|
|
|
* @return string |
500
|
|
|
*/ |
501
|
|
|
public function cleanHtml(string $html, string $expression = "//@*[local-name() != 'href' and local-name() != 'id' and local-name() != 'src']"): string |
502
|
|
|
{ |
503
|
|
|
$dom = new DOMDocument(); |
504
|
|
|
$metaUtf8 = '<meta http-equiv="content-type" content="text/html; charset=utf-8">'; |
505
|
|
|
$dom->loadHTML($metaUtf8 . $html, LIBXML_NOWARNING); |
506
|
|
|
$xpath = new DOMXPath($dom); |
507
|
|
|
$nodes = $xpath->query($expression); |
508
|
|
|
foreach ($nodes as $node) { |
509
|
|
|
/** @var \DOMElement $element */ |
510
|
|
|
$element = $node->parentNode; |
511
|
|
|
$element->removeAttribute($node->nodeName); |
512
|
|
|
} |
513
|
|
|
$body = $dom->documentElement->lastChild; |
514
|
|
|
$content = $dom->saveHTML($body); |
515
|
|
|
$content = preg_replace('/<\\/?body(\\s+.*?>|>)/', '', $content); |
516
|
|
|
|
517
|
|
|
return $content; |
518
|
|
|
} |
519
|
|
|
} |
520
|
|
|
|