1
|
|
|
<?php |
2
|
|
|
declare(strict_types=1); |
3
|
|
|
|
4
|
|
|
/** |
5
|
|
|
* BEdita, API-first content management framework |
6
|
|
|
* Copyright 2024 Atlas Srl, Chialab Srl |
7
|
|
|
* |
8
|
|
|
* This file is part of BEdita: you can redistribute it and/or modify |
9
|
|
|
* it under the terms of the GNU Lesser General Public License as published |
10
|
|
|
* by the Free Software Foundation, either version 3 of the License, or |
11
|
|
|
* (at your option) any later version. |
12
|
|
|
* |
13
|
|
|
* See LICENSE.LGPL or <http://gnu.org/licenses/lgpl-3.0.html> for more details. |
14
|
|
|
*/ |
15
|
|
|
namespace BEdita\ImportTools\Command; |
16
|
|
|
|
17
|
|
|
use BEdita\Core\Model\Entity\ObjectEntity; |
18
|
|
|
use BEdita\Core\Utility\JsonSchema; |
19
|
|
|
use BEdita\Core\Utility\LoggedUser; |
20
|
|
|
use Cake\Command\Command; |
21
|
|
|
use Cake\Console\Arguments; |
22
|
|
|
use Cake\Console\ConsoleIo; |
23
|
|
|
use Cake\Console\ConsoleOptionParser; |
24
|
|
|
use Cake\Core\Configure; |
25
|
|
|
use Cake\Core\InstanceConfigTrait; |
26
|
|
|
use Cake\Database\Expression\QueryExpression; |
27
|
|
|
use Cake\Utility\Hash; |
28
|
|
|
|
29
|
|
|
/** |
30
|
|
|
* TranslateObjects command. |
31
|
|
|
* Translate objects from a language to another using a translator engine. |
32
|
|
|
* The translator engine is defined in the configuration. |
33
|
|
|
* The configuration must contain the translator engine class and options |
34
|
|
|
* I.e.: |
35
|
|
|
* 'Translators' => [ |
36
|
|
|
* 'deepl' => [ |
37
|
|
|
* 'name' => 'DeepL', |
38
|
|
|
* 'class' => '\BEdita\I18n\Deepl\Core\Translator', |
39
|
|
|
* 'options' => [ |
40
|
|
|
* 'auth_key' => '************', |
41
|
|
|
* ], |
42
|
|
|
* ], |
43
|
|
|
* ], |
44
|
|
|
*/ |
45
|
|
|
class TranslateObjectsCommand extends Command |
46
|
|
|
{ |
47
|
|
|
use InstanceConfigTrait; |
48
|
|
|
|
49
|
|
|
protected $_defaultConfig = [ |
50
|
|
|
'langsMap' => [ |
51
|
|
|
'en' => 'en-US', |
52
|
|
|
'it' => 'it', |
53
|
|
|
'de' => 'de', |
54
|
|
|
'es' => 'es', |
55
|
|
|
'fr' => 'fr', |
56
|
|
|
'pt' => 'pt-PT', |
57
|
|
|
], |
58
|
|
|
'status' => 'draft', |
59
|
|
|
'dryRun' => false, |
60
|
|
|
]; |
61
|
|
|
protected $ok; |
62
|
|
|
protected $error; |
63
|
|
|
protected $io; |
64
|
|
|
protected $dryRun; |
65
|
|
|
protected $defaultStatus; |
66
|
|
|
protected $translatableFields = []; |
67
|
|
|
protected $translator; |
68
|
|
|
protected $langsMap; |
69
|
|
|
protected $limit; |
70
|
|
|
protected $type; |
71
|
|
|
|
72
|
|
|
/** |
73
|
|
|
* @inheritDoc |
74
|
|
|
*/ |
75
|
|
|
public function __construct() |
76
|
|
|
{ |
77
|
|
|
parent::__construct(); |
78
|
|
|
$cfg = (array)Configure::read('TranslateObjects'); |
79
|
|
|
$cfg = array_merge($this->_defaultConfig, $cfg); |
80
|
|
|
$this->defaultStatus = (string)Hash::get($cfg, 'status'); |
81
|
|
|
$this->setDryRun(Hash::get($cfg, 'dryRun') === 1); |
82
|
|
|
$this->langsMap = (array)Hash::get($cfg, 'langsMap'); |
83
|
|
|
} |
84
|
|
|
|
85
|
|
|
/** |
86
|
|
|
* @inheritDoc |
87
|
|
|
*/ |
88
|
|
|
public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser |
89
|
|
|
{ |
90
|
|
|
$langs = array_keys($this->langsMap); |
91
|
|
|
$parser = parent::buildOptionParser($parser); |
92
|
|
|
$parser->addOption('from', [ |
93
|
|
|
'short' => 'f', |
94
|
|
|
'help' => 'Language to translate from', |
95
|
|
|
'required' => true, |
96
|
|
|
'choices' => $langs, |
97
|
|
|
]); |
98
|
|
|
$parser->addOption('to', [ |
99
|
|
|
'short' => 't', |
100
|
|
|
'help' => 'Language to translate to', |
101
|
|
|
'required' => true, |
102
|
|
|
'choices' => $langs, |
103
|
|
|
]); |
104
|
|
|
$parser->addOption('engine', [ |
105
|
|
|
'short' => 'e', |
106
|
|
|
'help' => 'Translator engine', |
107
|
|
|
'default' => 'deepl', |
108
|
|
|
'required' => true, |
109
|
|
|
'choices' => ['aws', 'deepl', 'google', 'microsoft'], |
110
|
|
|
]); |
111
|
|
|
$parser->addOption('status', [ |
112
|
|
|
'short' => 's', |
113
|
|
|
'help' => 'Status for new translations', |
114
|
|
|
'choices' => ['draft', 'on', 'off'], |
115
|
|
|
'default' => 'draft', |
116
|
|
|
]); |
117
|
|
|
$parser->addOption('dry-run', [ |
118
|
|
|
'short' => 'd', |
119
|
|
|
'help' => 'Dry run', |
120
|
|
|
'boolean' => true, |
121
|
|
|
'default' => false, |
122
|
|
|
]); |
123
|
|
|
$parser->addOption('limit', [ |
124
|
|
|
'short' => 'l', |
125
|
|
|
'help' => 'Limit number of objects to translate', |
126
|
|
|
]); |
127
|
|
|
$parser->addOption('object-type', [ |
128
|
|
|
'short' => 'o', |
129
|
|
|
'help' => 'Object type to translate', |
130
|
|
|
]); |
131
|
|
|
|
132
|
|
|
return $parser; |
133
|
|
|
} |
134
|
|
|
|
135
|
|
|
/** |
136
|
|
|
* @inheritDoc |
137
|
|
|
*/ |
138
|
|
|
public function execute(Arguments $args, ConsoleIo $io): void |
139
|
|
|
{ |
140
|
|
|
$this->setDryRun($args->getOption('dry-run')); |
141
|
|
|
$this->setIo($io); |
142
|
|
|
|
143
|
|
|
// parameter engine |
144
|
|
|
$engine = $args->getOption('engine') ?? 'deepl'; |
145
|
|
|
|
146
|
|
|
// setup translator engine |
147
|
|
|
$cfg = Configure::read(sprintf('Translators.%s', $engine)); |
148
|
|
|
if (empty($cfg)) { |
149
|
|
|
$this->getIo()->abort(sprintf('Translator %s not found', $engine)); |
150
|
|
|
} |
151
|
|
|
$this->setTranslator($cfg); |
152
|
|
|
|
153
|
|
|
// parameters: lang from, lang to |
154
|
|
|
$from = $args->getOption('from'); |
155
|
|
|
$to = $args->getOption('to'); |
156
|
|
|
$this->limit = $args->getOption('limit') ?? null; |
157
|
|
|
$this->type = $args->getOption('object-type') ?? null; |
158
|
|
|
$this->getIo()->out( |
159
|
|
|
sprintf( |
160
|
|
|
'Translating objects from %s to %s [dry-run %s / limit %s / %s]', |
161
|
|
|
$from, |
162
|
|
|
$to, |
163
|
|
|
$this->dryRun ? 'yes' : 'no', |
164
|
|
|
$this->limit ? sprintf('limit %s', $this->limit) : 'unlimited', |
165
|
|
|
$this->type ? sprintf('type %s', $this->type) : 'all types' |
166
|
|
|
) |
167
|
|
|
); |
168
|
|
|
$to = $this->langsMap[$to]; |
169
|
|
|
if ($this->getIo()->ask('Do you want to continue [Y/n]?', 'n') !== 'Y') { |
170
|
|
|
$this->getIo()->abort('Bye'); |
171
|
|
|
} |
172
|
|
|
$this->ok = $this->error = 0; |
173
|
|
|
$this->defaultStatus = $args->getOption('status'); |
174
|
|
|
$this->processObjects($from, $to); |
175
|
|
|
$this->getIo()->out($this->results()); |
176
|
|
|
$this->getIo()->out('Done'); |
177
|
|
|
} |
178
|
|
|
|
179
|
|
|
/** |
180
|
|
|
* Set console io. |
181
|
|
|
* |
182
|
|
|
* @param \Cake\Console\ConsoleIo $io The console io |
183
|
|
|
* @return void |
184
|
|
|
*/ |
185
|
|
|
public function setIo(ConsoleIo $io): void |
186
|
|
|
{ |
187
|
|
|
$this->io = $io; |
188
|
|
|
} |
189
|
|
|
|
190
|
|
|
/** |
191
|
|
|
* Get console io. |
192
|
|
|
* |
193
|
|
|
* @return \Cake\Console\ConsoleIo |
194
|
|
|
*/ |
195
|
|
|
public function getIo(): ConsoleIo |
196
|
|
|
{ |
197
|
|
|
return $this->io; |
198
|
|
|
} |
199
|
|
|
|
200
|
|
|
/** |
201
|
|
|
* Set dry run. |
202
|
|
|
* |
203
|
|
|
* @param bool $dryRun The dry run flag |
204
|
|
|
* @return void |
205
|
|
|
*/ |
206
|
|
|
public function setDryRun(bool $dryRun): void |
207
|
|
|
{ |
208
|
|
|
$this->dryRun = $dryRun; |
209
|
|
|
} |
210
|
|
|
|
211
|
|
|
/** |
212
|
|
|
* Get dry run. |
213
|
|
|
* |
214
|
|
|
* @return bool |
215
|
|
|
*/ |
216
|
|
|
public function getDryRun(): bool |
217
|
|
|
{ |
218
|
|
|
return $this->dryRun; |
219
|
|
|
} |
220
|
|
|
|
221
|
|
|
/** |
222
|
|
|
* Get results. |
223
|
|
|
* |
224
|
|
|
* @return string |
225
|
|
|
*/ |
226
|
|
|
public function results(): string |
227
|
|
|
{ |
228
|
|
|
return sprintf('Processed %d objects (%d errors)', $this->ok + $this->error, $this->error); |
229
|
|
|
} |
230
|
|
|
|
231
|
|
|
/** |
232
|
|
|
* Set translator engine. |
233
|
|
|
* |
234
|
|
|
* @param array $cfg The translator configuration |
235
|
|
|
* @return void |
236
|
|
|
*/ |
237
|
|
|
public function setTranslator(array $cfg): void |
238
|
|
|
{ |
239
|
|
|
$class = (string)Hash::get($cfg, 'class'); |
240
|
|
|
$options = (array)Hash::get($cfg, 'options'); |
241
|
|
|
$this->translator = new $class(); |
242
|
|
|
$this->translator->setup($options); |
243
|
|
|
} |
244
|
|
|
|
245
|
|
|
/** |
246
|
|
|
* Process objects to translate. |
247
|
|
|
* |
248
|
|
|
* @param string $from The language to translate from |
249
|
|
|
* @param string $to The language to translate to |
250
|
|
|
* @return void |
251
|
|
|
*/ |
252
|
|
|
public function processObjects(string $from, string $to): void |
253
|
|
|
{ |
254
|
|
|
$conditions = []; |
255
|
|
|
foreach ($this->objectsIterator($conditions, $from, $to) as $object) { |
256
|
|
|
if ($this->limit !== null && ($this->ok + $this->error >= $this->limit)) { |
257
|
|
|
break; |
258
|
|
|
} |
259
|
|
|
$this->processObject($object, $from, $to); |
260
|
|
|
} |
261
|
|
|
} |
262
|
|
|
|
263
|
|
|
/** |
264
|
|
|
* Process single object. |
265
|
|
|
* |
266
|
|
|
* @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate |
267
|
|
|
* @param string $from The language to translate from |
268
|
|
|
* @param string $to The language to translate to |
269
|
|
|
* @return void |
270
|
|
|
*/ |
271
|
|
|
public function processObject(ObjectEntity $object, string $from, string $to): void |
272
|
|
|
{ |
273
|
|
|
try { |
274
|
|
|
$this->getIo()->verbose(sprintf('Translating object %s', $object->id)); |
275
|
|
|
if (!$this->dryRun) { |
276
|
|
|
$this->translate($object, $from, $to); |
277
|
|
|
} |
278
|
|
|
$this->getIo()->verbose(sprintf('Translated object %s', $object->id)); |
279
|
|
|
$this->ok++; |
280
|
|
|
} catch (\Exception $e) { |
281
|
|
|
$this->getIo()->error(sprintf('Error translating object %s: %s', $object->id, $e->getMessage())); |
282
|
|
|
$this->error++; |
283
|
|
|
} |
284
|
|
|
} |
285
|
|
|
|
286
|
|
|
/** |
287
|
|
|
* Get objects as iterable. |
288
|
|
|
* |
289
|
|
|
* @param array $conditions The conditions to filter objects. |
290
|
|
|
* @param string $lang The language to use to find objects. |
291
|
|
|
* @param string $to The language to translate objects to. |
292
|
|
|
* @return iterable |
293
|
|
|
*/ |
294
|
|
|
public function objectsIterator(array $conditions, string $lang, string $to): iterable |
295
|
|
|
{ |
296
|
|
|
/** @var \BEdita\Core\Model\Table\ObjectsTable $table */ |
297
|
|
|
$table = $this->fetchTable('objects'); |
298
|
|
|
if ($this->type !== null) { |
299
|
|
|
$conditions[$table->aliasField('object_type_id')] = $table->objectType($this->type)->id; |
300
|
|
|
} |
301
|
|
|
$conditions = array_merge( |
302
|
|
|
$conditions, |
303
|
|
|
[ |
304
|
|
|
$table->aliasField('deleted') => 0, |
305
|
|
|
$table->aliasField('lang') => $lang, |
306
|
|
|
] |
307
|
|
|
); |
308
|
|
|
$query = $table->find('all') |
309
|
|
|
->where($conditions) |
310
|
|
|
->notMatching('Translations', function ($q) use ($to) { |
311
|
|
|
return $q->where(['Translations.lang' => $to]); |
312
|
|
|
}) |
313
|
|
|
->orderAsc($table->aliasField('id')) |
314
|
|
|
->limit(500); |
315
|
|
|
$lastId = 0; |
316
|
|
|
while (true) { |
317
|
|
|
$q = clone $query; |
318
|
|
|
$q = $q->where(fn (QueryExpression $exp): QueryExpression => $exp->gt($table->aliasField('id'), $lastId)); |
319
|
|
|
$results = $q->all(); |
320
|
|
|
if ($results->isEmpty()) { |
321
|
|
|
break; |
322
|
|
|
} |
323
|
|
|
|
324
|
|
|
foreach ($results as $entity) { |
325
|
|
|
$lastId = $entity->id; |
326
|
|
|
|
327
|
|
|
yield $entity; |
328
|
|
|
} |
329
|
|
|
} |
330
|
|
|
} |
331
|
|
|
|
332
|
|
|
/** |
333
|
|
|
* Translate object translatable fields and store translation in translations table. |
334
|
|
|
* |
335
|
|
|
* @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate |
336
|
|
|
* @param string $from The language to translate from |
337
|
|
|
* @param string $to The language to translate to |
338
|
|
|
* @return void |
339
|
|
|
*/ |
340
|
|
|
public function translate($object, $from, $to): void |
341
|
|
|
{ |
342
|
|
|
$id = $object->id; |
343
|
|
|
$type = $object->type; |
344
|
|
|
$object = $this->fetchTable($type)->find()->where(compact('id'))->firstOrFail(); |
345
|
|
|
$translatableFields = $this->translatableFields($type); |
346
|
|
|
if (empty($translatableFields)) { |
347
|
|
|
return; |
348
|
|
|
} |
349
|
|
|
$translatedFields = []; |
350
|
|
|
$fields = $values = $jsonFields = $jsonValues = []; |
351
|
|
|
foreach ($translatableFields as $field) { |
352
|
|
|
if (empty($object->get($field))) { |
353
|
|
|
continue; |
354
|
|
|
} |
355
|
|
|
$val = $object->get($field); |
356
|
|
|
if (is_array($val)) { |
357
|
|
|
$jsonFields[] = $field; |
358
|
|
|
$jsonValues[] = json_encode($val); |
359
|
|
|
} else { |
360
|
|
|
$fields[] = $field; |
361
|
|
|
$values[] = $val; |
362
|
|
|
} |
363
|
|
|
} |
364
|
|
|
if (empty($fields) && empty($jsonFields)) { |
365
|
|
|
return; |
366
|
|
|
} |
367
|
|
|
$tr = $this->multiTranslation($values, $from, $to); |
368
|
|
|
foreach ($tr as $i => $t) { |
369
|
|
|
$translatedFields[$fields[$i]] = $t; |
370
|
|
|
} |
371
|
|
|
$tr = $this->multiTranslation($jsonValues, $from, $to); |
372
|
|
|
foreach ($tr as $i => $t) { |
373
|
|
|
$translatedFields[$jsonFields[$i]] = json_decode($t, true); |
374
|
|
|
} |
375
|
|
|
$translation = [ |
376
|
|
|
'object_id' => $id, |
377
|
|
|
'lang' => array_flip($this->langsMap)[$to], |
378
|
|
|
'translated_fields' => json_encode($translatedFields), |
379
|
|
|
'status' => $this->defaultStatus, |
380
|
|
|
]; |
381
|
|
|
$table = $this->fetchTable('Translations'); |
382
|
|
|
$entity = $table->newEntity($translation); |
383
|
|
|
$entity->set('translated_fields', json_decode($translation['translated_fields'], true)); |
384
|
|
|
$entity->set('status', $this->defaultStatus); |
385
|
|
|
LoggedUser::setUserAdmin(); |
386
|
|
|
$table->saveOrFail($entity); |
387
|
|
|
} |
388
|
|
|
|
389
|
|
|
/** |
390
|
|
|
* Get translatable fields by object type. |
391
|
|
|
* |
392
|
|
|
* @param string $type The object type |
393
|
|
|
* @return array |
394
|
|
|
*/ |
395
|
|
|
public function translatableFields(string $type): array |
396
|
|
|
{ |
397
|
|
|
if (array_key_exists($type, $this->translatableFields)) { |
398
|
|
|
return $this->translatableFields[$type]; |
399
|
|
|
} |
400
|
|
|
$schema = JsonSchema::typeSchema($type); |
401
|
|
|
$this->translatableFields[$type] = (array)Hash::get($schema, 'translatable'); |
402
|
|
|
|
403
|
|
|
return $this->translatableFields[$type]; |
404
|
|
|
} |
405
|
|
|
|
406
|
|
|
/** |
407
|
|
|
* Translate a single text. |
408
|
|
|
* |
409
|
|
|
* @param mixed $text The text to translate |
410
|
|
|
* @param string $from The language to translate from |
411
|
|
|
* @param string $to The language to translate to |
412
|
|
|
* @return string |
413
|
|
|
*/ |
414
|
|
|
public function singleTranslation($text, string $from, string $to): string |
415
|
|
|
{ |
416
|
|
|
$response = $this->translator->translate([$text], $from, $to); |
417
|
|
|
$response = json_decode($response, true); |
418
|
|
|
|
419
|
|
|
return (string)Hash::get($response, 'translation.0'); |
420
|
|
|
} |
421
|
|
|
|
422
|
|
|
/** |
423
|
|
|
* Translate multiple texts. |
424
|
|
|
* |
425
|
|
|
* @param array $texts The texts to translate |
426
|
|
|
* @param string $from The language to translate from |
427
|
|
|
* @param string $to The language to translate to |
428
|
|
|
* @return array |
429
|
|
|
*/ |
430
|
|
|
public function multiTranslation($texts, string $from, string $to): array |
431
|
|
|
{ |
432
|
|
|
$response = $this->translator->translate($texts, $from, $to); |
433
|
|
|
$response = json_decode($response, true); |
434
|
|
|
|
435
|
|
|
return (array)Hash::get($response, 'translation'); |
436
|
|
|
} |
437
|
|
|
} |
438
|
|
|
|