TranslateObjectsCommand::objectsIterator()   A
last analyzed

Complexity

Conditions 5
Paths 8

Size

Total Lines 34
Code Lines 23

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 0 Features 0
Metric Value
eloc 23
c 1
b 0
f 0
dl 0
loc 34
rs 9.2408
cc 5
nc 8
nop 3
1
<?php
2
declare(strict_types=1);
3
4
/**
5
 * BEdita, API-first content management framework
6
 * Copyright 2024 Atlas Srl, Chialab Srl
7
 *
8
 * This file is part of BEdita: you can redistribute it and/or modify
9
 * it under the terms of the GNU Lesser General Public License as published
10
 * by the Free Software Foundation, either version 3 of the License, or
11
 * (at your option) any later version.
12
 *
13
 * See LICENSE.LGPL or <http://gnu.org/licenses/lgpl-3.0.html> for more details.
14
 */
15
namespace BEdita\ImportTools\Command;
16
17
use BEdita\Core\Model\Entity\ObjectEntity;
18
use BEdita\Core\Utility\JsonSchema;
19
use BEdita\Core\Utility\LoggedUser;
20
use Cake\Command\Command;
21
use Cake\Console\Arguments;
22
use Cake\Console\ConsoleIo;
23
use Cake\Console\ConsoleOptionParser;
24
use Cake\Core\Configure;
25
use Cake\Core\InstanceConfigTrait;
26
use Cake\Database\Expression\QueryExpression;
27
use Cake\Utility\Hash;
28
use Exception;
29
30
/**
31
 * TranslateObjects command.
32
 *
33
 * $ bin/cake translate_objects --help
34
 * $ bin/cake translate_objects \
35
 *  --from source_language \
36
 *  --to dest_language \
37
 *  --engine translator_engine \
38
 * [--status translation_status] \
39
 * [--dry-run] \
40
 * [--limit number_of_objects] \
41
 * [--object-type object_type]
42
 *
43
 * Translate objects from a language to another using a translator engine.
44
 * The translator engine is defined in the configuration.
45
 * The configuration must contain the translator engine class and options
46
 * I.e.:
47
 * 'Translators' => [
48
 *   'deepl' => [
49
 *      'name' => 'DeepL',
50
 *      'class' => '\BEdita\I18n\Deepl\Core\Translator',
51
 *      'options' => [
52
 *        'auth_key' => '************',
53
 *      ],
54
 *   ],
55
 * ],
56
 */
57
class TranslateObjectsCommand extends Command
58
{
59
    use InstanceConfigTrait;
60
61
    protected array $_defaultConfig = [
62
        'langsMap' => [
63
            'en' => 'en-US',
64
            'it' => 'it',
65
            'de' => 'de',
66
            'es' => 'es',
67
            'fr' => 'fr',
68
            'pt' => 'pt-PT',
69
        ],
70
        'status' => 'draft',
71
        'dryRun' => false,
72
    ];
73
    protected int $ok = 0;
74
    protected int $error = 0;
75
    protected ConsoleIo $io;
76
    protected bool $dryRun = false;
77
    protected string $defaultStatus;
78
    protected array $translatableFields = [];
79
    protected object $translator;
80
    protected array $langsMap;
81
    protected ?string $limit = null;
82
    protected ?string $type = null;
83
84
    /**
85
     * @inheritDoc
86
     */
87
    public function __construct()
88
    {
89
        parent::__construct();
90
        $cfg = (array)Configure::read('TranslateObjects');
91
        $cfg = array_merge($this->_defaultConfig, $cfg);
92
        $this->defaultStatus = Hash::get($cfg, 'status');
93
        $this->setDryRun(Hash::get($cfg, 'dryRun') === 1);
94
        $this->langsMap = (array)Hash::get($cfg, 'langsMap');
95
    }
96
97
    /**
98
     * @inheritDoc
99
     */
100
    public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser
101
    {
102
        $langs = array_keys($this->langsMap);
103
        $parser = parent::buildOptionParser($parser);
104
        $parser->addOption('from', [
105
            'short' => 'f',
106
            'help' => 'Language to translate from',
107
            'required' => true,
108
            'choices' => $langs,
109
        ]);
110
        $parser->addOption('to', [
111
            'short' => 't',
112
            'help' => 'Language to translate to',
113
            'required' => true,
114
            'choices' => $langs,
115
        ]);
116
        $parser->addOption('engine', [
117
            'short' => 'e',
118
            'help' => 'Translator engine',
119
            'default' => 'deepl',
120
            'required' => true,
121
            'choices' => ['aws', 'deepl', 'google', 'microsoft'],
122
        ]);
123
        $parser->addOption('status', [
124
            'short' => 's',
125
            'help' => 'Status for new translations',
126
            'choices' => ['draft', 'on', 'off'],
127
            'default' => 'draft',
128
        ]);
129
        $parser->addOption('dry-run', [
130
            'short' => 'd',
131
            'help' => 'Dry run',
132
            'boolean' => true,
133
            'default' => false,
134
        ]);
135
        $parser->addOption('limit', [
136
            'short' => 'l',
137
            'help' => 'Limit number of objects to translate',
138
        ]);
139
        $parser->addOption('object-type', [
140
            'short' => 'o',
141
            'help' => 'Object type to translate',
142
        ]);
143
144
        return $parser;
145
    }
146
147
    /**
148
     * @inheritDoc
149
     */
150
    public function execute(Arguments $args, ConsoleIo $io): void
151
    {
152
        $this->setDryRun($args->getOption('dry-run'));
153
        $this->setIo($io);
154
155
        // parameter engine
156
        $engine = $args->getOption('engine') ?? 'deepl';
157
158
        // setup translator engine
159
        $cfg = Configure::read(sprintf('Translators.%s', $engine));
160
        if (empty($cfg)) {
161
            $this->getIo()->abort(sprintf('Translator %s not found', $engine));
162
        }
163
        $this->setTranslator($cfg);
164
165
        // parameters: lang from, lang to
166
        $from = $args->getOption('from');
167
        $to = $args->getOption('to');
168
        $this->limit = $args->getOption('limit') ?? null;
169
        $this->type = $args->getOption('object-type') ?? null;
170
        $this->getIo()->out(
171
            sprintf(
172
                'Translating objects from %s to %s [dry-run %s / limit %s / %s]',
173
                $from,
174
                $to,
175
                $this->dryRun ? 'yes' : 'no',
176
                $this->limit ? sprintf('limit %s', $this->limit) : 'unlimited',
177
                $this->type ? sprintf('type %s', $this->type) : 'all types',
178
            ),
179
        );
180
        $to = $this->langsMap[$to];
181
        if ($this->getIo()->ask('Do you want to continue [Y/n]?', 'n') !== 'Y') {
182
            $this->getIo()->abort('Bye');
183
        }
184
        $this->ok = $this->error = 0;
185
        $this->defaultStatus = $args->getOption('status');
186
        $this->processObjects($from, $to);
187
        $this->getIo()->out($this->results());
188
        $this->getIo()->out('Done');
189
    }
190
191
    /**
192
     * Set console io.
193
     *
194
     * @param \Cake\Console\ConsoleIo $io The console io
195
     * @return void
196
     */
197
    public function setIo(ConsoleIo $io): void
198
    {
199
        $this->io = $io;
200
    }
201
202
    /**
203
     * Get console io.
204
     *
205
     * @return \Cake\Console\ConsoleIo
206
     */
207
    public function getIo(): ConsoleIo
208
    {
209
        return $this->io;
210
    }
211
212
    /**
213
     * Set dry run.
214
     *
215
     * @param bool $dryRun The dry run flag
216
     * @return void
217
     */
218
    public function setDryRun(bool $dryRun): void
219
    {
220
        $this->dryRun = $dryRun;
221
    }
222
223
    /**
224
     * Get dry run.
225
     *
226
     * @return bool
227
     */
228
    public function getDryRun(): bool
229
    {
230
        return $this->dryRun;
231
    }
232
233
    /**
234
     * Get results.
235
     *
236
     * @return string
237
     */
238
    public function results(): string
239
    {
240
        return sprintf('Processed %d objects (%d errors)', $this->ok + $this->error, $this->error);
241
    }
242
243
    /**
244
     * Set translator engine.
245
     *
246
     * @param array $cfg The translator configuration
247
     * @return void
248
     */
249
    public function setTranslator(array $cfg): void
250
    {
251
        $class = (string)Hash::get($cfg, 'class');
252
        $options = (array)Hash::get($cfg, 'options');
253
        $this->translator = new $class();
254
        $this->translator->setup($options);
255
    }
256
257
    /**
258
     * Process objects to translate.
259
     *
260
     * @param string $from The language to translate from
261
     * @param string $to The language to translate to
262
     * @return void
263
     */
264
    public function processObjects(string $from, string $to): void
265
    {
266
        $conditions = [];
267
        foreach ($this->objectsIterator($conditions, $from, $to) as $object) {
268
            if ($this->limit !== null && ($this->ok + $this->error >= intval($this->limit))) {
269
                break;
270
            }
271
            $this->processObject($object, $from, $to);
272
        }
273
    }
274
275
    /**
276
     * Process single object.
277
     *
278
     * @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate
279
     * @param string $from The language to translate from
280
     * @param string $to The language to translate to
281
     * @return void
282
     */
283
    public function processObject(ObjectEntity $object, string $from, string $to): void
284
    {
285
        try {
286
            $this->getIo()->verbose(sprintf('Translating object %s', $object->id));
287
            if (!$this->dryRun) {
288
                $this->translate($object, $from, $to);
289
            }
290
            $this->getIo()->verbose(sprintf('Translated object %s', $object->id));
291
            $this->ok++;
292
        } catch (Exception $e) {
293
            $this->getIo()->error(sprintf('Error translating object %s: %s', $object->id, $e->getMessage()));
294
            $this->error++;
295
        }
296
    }
297
298
    /**
299
     * Get objects as iterable.
300
     *
301
     * @param array $conditions The conditions to filter objects.
302
     * @param string $lang The language to use to find objects.
303
     * @param string $to The language to translate objects to.
304
     * @return iterable
305
     */
306
    public function objectsIterator(array $conditions, string $lang, string $to): iterable
307
    {
308
        /** @var \BEdita\Core\Model\Table\ObjectsTable $table */
309
        $table = $this->fetchTable('objects');
310
        if ($this->type !== null) {
311
            $conditions[$table->aliasField('object_type_id')] = $table->objectType($this->type)->id;
312
        }
313
        $conditions = array_merge(
314
            $conditions,
315
            [
316
                $table->aliasField('deleted') => 0,
317
                $table->aliasField('lang') => $lang,
318
            ],
319
        );
320
        $query = $table->find('all')
321
            ->where($conditions)
322
            ->notMatching('Translations', function ($q) use ($to) {
323
                return $q->where(['Translations.lang' => $to]);
324
            })
325
            ->orderByAsc($table->aliasField('id'))
326
            ->limit(500);
327
        $lastId = 0;
328
        while (true) {
329
            $q = clone $query;
330
            $q = $q->where(fn(QueryExpression $exp): QueryExpression => $exp->gt($table->aliasField('id'), $lastId));
331
            $results = $q->all();
332
            if ($results->isEmpty()) {
333
                break;
334
            }
335
336
            foreach ($results as $entity) {
337
                $lastId = $entity->id;
338
339
                yield $entity;
340
            }
341
        }
342
    }
343
344
    /**
345
     * Translate object translatable fields and store translation in translations table.
346
     *
347
     * @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate
348
     * @param string $from The language to translate from
349
     * @param string $to The language to translate to
350
     * @return void
351
     */
352
    public function translate(ObjectEntity $object, string $from, string $to): void
353
    {
354
        $id = $object->id;
355
        $type = $object->type;
356
        $object = $this->fetchTable($type)->find()->where(compact('id'))->firstOrFail();
357
        $translatableFields = $this->translatableFields($type);
358
        if (empty($translatableFields)) {
359
            return;
360
        }
361
        $translatedFields = [];
362
        $fields = $values = $jsonFields = $jsonValues = [];
363
        foreach ($translatableFields as $field) {
364
            if (empty($object->get($field))) {
365
                continue;
366
            }
367
            $val = $object->get($field);
368
            if (is_array($val)) {
369
                $jsonFields[] = $field;
370
                $jsonValues[] = json_encode($val);
371
            } else {
372
                $fields[] = $field;
373
                $values[] = $val;
374
            }
375
        }
376
        if (empty($fields) && empty($jsonFields)) {
377
            return;
378
        }
379
        $tr = $this->multiTranslation($values, $from, $to);
380
        foreach ($tr as $i => $t) {
381
            $translatedFields[$fields[$i]] = $t;
382
        }
383
        $tr = $this->multiTranslation($jsonValues, $from, $to);
384
        foreach ($tr as $i => $t) {
385
            $translatedFields[$jsonFields[$i]] = json_decode($t, true);
386
        }
387
        $translation = [
388
            'object_id' => $id,
389
            'lang' => array_flip($this->langsMap)[$to],
390
            'translated_fields' => json_encode($translatedFields),
391
            'status' => $this->defaultStatus,
392
        ];
393
        $table = $this->fetchTable('Translations');
394
        $entity = $table->newEntity($translation);
395
        $entity->set('translated_fields', json_decode($translation['translated_fields'], true));
396
        $entity->set('status', $this->defaultStatus);
397
        LoggedUser::setUserAdmin();
398
        $table->saveOrFail($entity);
399
    }
400
401
    /**
402
     * Get translatable fields by object type.
403
     *
404
     * @param string $type The object type
405
     * @return array
406
     */
407
    public function translatableFields(string $type): array
408
    {
409
        if (array_key_exists($type, $this->translatableFields)) {
410
            return $this->translatableFields[$type];
411
        }
412
        $schema = JsonSchema::typeSchema($type);
413
        $this->translatableFields[$type] = (array)Hash::get($schema, 'translatable');
414
415
        return $this->translatableFields[$type];
416
    }
417
418
    /**
419
     * Translate a single text.
420
     *
421
     * @param mixed $text The text to translate
422
     * @param string $from The language to translate from
423
     * @param string $to The language to translate to
424
     * @return string
425
     */
426
    public function singleTranslation(mixed $text, string $from, string $to): string
427
    {
428
        $response = $this->translator->translate([$text], $from, $to);
429
        $response = json_decode($response, true);
430
431
        return (string)Hash::get($response, 'translation.0');
432
    }
433
434
    /**
435
     * Translate multiple texts.
436
     *
437
     * @param array $texts The texts to translate
438
     * @param string $from The language to translate from
439
     * @param string $to The language to translate to
440
     * @return array
441
     */
442
    public function multiTranslation(array $texts, string $from, string $to): array
443
    {
444
        $response = $this->translator->translate($texts, $from, $to);
445
        $response = json_decode($response, true);
446
447
        return (array)Hash::get($response, 'translation');
448
    }
449
}
450