Passed
Pull Request — main (#9)
by Dante
01:08
created

TranslateObjectsCommand::translate()   B

Complexity

Conditions 9
Paths 21

Size

Total Lines 47
Code Lines 37

Duplication

Lines 0
Ratio 0 %

Importance

Changes 4
Bugs 2 Features 0
Metric Value
eloc 37
c 4
b 2
f 0
dl 0
loc 47
rs 7.7724
cc 9
nc 21
nop 3
1
<?php
2
declare(strict_types=1);
3
4
/**
5
 * BEdita, API-first content management framework
6
 * Copyright 2024 Atlas Srl, Chialab Srl
7
 *
8
 * This file is part of BEdita: you can redistribute it and/or modify
9
 * it under the terms of the GNU Lesser General Public License as published
10
 * by the Free Software Foundation, either version 3 of the License, or
11
 * (at your option) any later version.
12
 *
13
 * See LICENSE.LGPL or <http://gnu.org/licenses/lgpl-3.0.html> for more details.
14
 */
15
namespace BEdita\ImportTools\Command;
16
17
use BEdita\Core\Model\Entity\ObjectEntity;
18
use BEdita\Core\Model\Table\ObjectsTable;
19
use BEdita\Core\Utility\JsonSchema;
20
use BEdita\Core\Utility\LoggedUser;
21
use Cake\Command\Command;
22
use Cake\Console\Arguments;
23
use Cake\Console\ConsoleIo;
24
use Cake\Console\ConsoleOptionParser;
25
use Cake\Core\Configure;
26
use Cake\Core\InstanceConfigTrait;
27
use Cake\Database\Expression\QueryExpression;
28
use Cake\Utility\Hash;
29
30
/**
31
 * TranslateObjects command.
32
 * Translate objects from a language to another using a translator engine.
33
 * The translator engine is defined in the configuration.
34
 * The configuration must contain the translator engine class and options
35
 * I.e.:
36
 * 'Translators' => [
37
 *   'deepl' => [
38
 *      'name' => 'DeepL',
39
 *      'class' => '\BEdita\I18n\Deepl\Core\Translator',
40
 *      'options' => [
41
 *        'auth_key' => '************',
42
 *      ],
43
 *   ],
44
 * ],
45
 */
46
class TranslateObjectsCommand extends Command
47
{
48
    use InstanceConfigTrait;
49
50
    protected $_defaultConfig = [
51
        'langsMap' => [
52
            'en' => 'en-US',
53
            'it' => 'it',
54
            'de' => 'de',
55
            'es' => 'es',
56
            'fr' => 'fr',
57
            'pt' => 'pt-PT',
58
        ],
59
        'status' => 'draft',
60
        'dryRun' => false,
61
    ];
62
    protected $ok;
63
    protected $error;
64
    protected $io;
65
    protected $dryRun;
66
    protected $defaultStatus;
67
    protected $translatableFields = [];
68
    protected $translator;
69
    protected $langsMap;
70
    protected $limit;
71
    protected $type;
72
73
    /**
74
     * @inheritDoc
75
     */
76
    public function __construct()
77
    {
78
        parent::__construct();
79
        $cfg = (array)Configure::read('TranslateObjects');
80
        $cfg = array_merge($this->_defaultConfig, $cfg);
81
        $this->defaultStatus = (string)Hash::get($cfg, 'status');
82
        $this->setDryRun(Hash::get($cfg, 'dryRun') === 1);
83
        $this->langsMap = (array)Hash::get($cfg, 'langsMap');
84
    }
85
86
    /**
87
     * @inheritDoc
88
     */
89
    public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser
90
    {
91
        $langs = array_keys($this->langsMap);
92
        $parser = parent::buildOptionParser($parser);
93
        $parser->addOption('from', [
94
            'short' => 'f',
95
            'help' => 'Language to translate from',
96
            'required' => true,
97
            'choices' => $langs,
98
        ]);
99
        $parser->addOption('to', [
100
            'short' => 't',
101
            'help' => 'Language to translate to',
102
            'required' => true,
103
            'choices' => $langs,
104
        ]);
105
        $parser->addOption('engine', [
106
            'short' => 'e',
107
            'help' => 'Translator engine',
108
            'default' => 'deepl',
109
            'required' => true,
110
            'choices' => ['aws', 'deepl', 'google', 'microsoft'],
111
        ]);
112
        $parser->addOption('status', [
113
            'short' => 's',
114
            'help' => 'Status for new translations',
115
            'choices' => ['draft', 'on', 'off'],
116
            'default' => 'draft',
117
        ]);
118
        $parser->addOption('dry-run', [
119
            'short' => 'd',
120
            'help' => 'Dry run',
121
            'boolean' => true,
122
            'default' => false,
123
        ]);
124
        $parser->addOption('limit', [
125
            'short' => 'l',
126
            'help' => 'Limit number of objects to translate',
127
        ]);
128
        $parser->addOption('object-type', [
129
            'short' => 'o',
130
            'help' => 'Object type to translate',
131
        ]);
132
133
        return $parser;
134
    }
135
136
    /**
137
     * @inheritDoc
138
     */
139
    public function execute(Arguments $args, ConsoleIo $io): void
140
    {
141
        $this->setDryRun($args->getOption('dry-run'));
142
        $this->setIo($io);
143
144
        // parameter engine
145
        $engine = $args->getOption('engine') ?? 'deepl';
146
147
        // setup translator engine
148
        $cfg = Configure::read(sprintf('Translators.%s', $engine));
149
        if (empty($cfg)) {
150
            $this->getIo()->abort(sprintf('Translator %s not found', $engine));
151
        }
152
        $this->setTranslator($cfg);
153
154
        // parameters: lang from, lang to
155
        $from = $args->getOption('from');
156
        $to = $args->getOption('to');
157
        $this->limit = $args->getOption('limit') ?? null;
158
        $this->type = $args->getOption('object-type') ?? null;
159
        $this->getIo()->out(
160
            sprintf(
161
                'Translating objects from %s to %s [dry-run %s / limit %s / %s]',
162
                $from,
163
                $to,
164
                $this->dryRun ? 'yes' : 'no',
165
                $this->limit ? sprintf('limit %s', $this->limit) : 'unlimited',
166
                $this->type ? sprintf('type %s', $this->type) : 'all types'
167
            )
168
        );
169
        $to = $this->langsMap[$to];
170
        if ($this->getIo()->ask('Do you want to continue [Y/n]?', 'n') !== 'Y') {
171
            $this->getIo()->abort('Bye');
172
        }
173
        $this->ok = $this->error = 0;
174
        $this->defaultStatus = $args->getOption('status');
175
        $this->processObjects($from, $to);
176
        $this->getIo()->out($this->results());
177
        $this->getIo()->out('Done');
178
    }
179
180
    /**
181
     * Set console io.
182
     *
183
     * @param \Cake\Console\ConsoleIo $io The console io
184
     * @return void
185
     */
186
    public function setIo(ConsoleIo $io): void
187
    {
188
        $this->io = $io;
189
    }
190
191
    /**
192
     * Get console io.
193
     *
194
     * @return \Cake\Console\ConsoleIo
195
     */
196
    public function getIo(): ConsoleIo
197
    {
198
        return $this->io;
199
    }
200
201
    /**
202
     * Set dry run.
203
     *
204
     * @param bool $dryRun The dry run flag
205
     * @return void
206
     */
207
    public function setDryRun(bool $dryRun): void
208
    {
209
        $this->dryRun = $dryRun;
210
    }
211
212
    /**
213
     * Get dry run.
214
     *
215
     * @return bool
216
     */
217
    public function getDryRun(): bool
218
    {
219
        return $this->dryRun;
220
    }
221
222
    /**
223
     * Get results.
224
     *
225
     * @return string
226
     */
227
    public function results(): string
228
    {
229
        return sprintf('Processed %d objects (%d errors)', $this->ok + $this->error, $this->error);
230
    }
231
232
    /**
233
     * Set translator engine.
234
     *
235
     * @param array $cfg The translator configuration
236
     * @return void
237
     */
238
    public function setTranslator(array $cfg): void
239
    {
240
        $class = (string)Hash::get($cfg, 'class');
241
        $options = (array)Hash::get($cfg, 'options');
242
        $this->translator = new $class();
243
        $this->translator->setup($options);
244
    }
245
246
    /**
247
     * Process objects to translate.
248
     *
249
     * @param string $from The language to translate from
250
     * @param string $to The language to translate to
251
     * @return void
252
     */
253
    public function processObjects(string $from, string $to): void
254
    {
255
        $conditions = [];
256
        foreach ($this->objectsIterator($conditions, $from, $to) as $object) {
257
            if ($this->limit !== null && ($this->ok + $this->error >= $this->limit)) {
258
                break;
259
            }
260
            $this->processObject($object, $from, $to);
261
        }
262
    }
263
264
    /**
265
     * Process single object.
266
     *
267
     * @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate
268
     * @param string $from The language to translate from
269
     * @param string $to The language to translate to
270
     * @return void
271
     */
272
    public function processObject(ObjectEntity $object, string $from, string $to): void
273
    {
274
        try {
275
            $this->getIo()->verbose(sprintf('Translating object %s', $object->id));
276
            if (!$this->dryRun) {
277
                $this->translate($object, $from, $to);
278
            }
279
            $this->getIo()->verbose(sprintf('Translated object %s', $object->id));
280
            $this->ok++;
281
        } catch (\Exception $e) {
282
            $this->getIo()->error(sprintf('Error translating object %s: %s', $object->id, $e->getMessage()));
283
            $this->error++;
284
        }
285
    }
286
287
    /**
288
     * Get objects as iterable.
289
     *
290
     * @param array $conditions The conditions to filter objects.
291
     * @param string $lang The language to use to find objects.
292
     * @param string $to The language to translate objects to.
293
     * @return iterable
294
     */
295
    public function objectsIterator(array $conditions, string $lang, string $to): iterable
296
    {
297
        /** @var \BEdita\Core\Model\Table\ObjectsTable $table */
298
        $table = $this->fetchTable('objects');
299
        if ($this->type !== null) {
300
            $conditions[$table->aliasField('object_type_id')] = $table->objectType($this->type)->id;
301
        }
302
        $conditions = array_merge(
303
            $conditions,
304
            [
305
                $table->aliasField('deleted') => 0,
306
                $table->aliasField('lang') => $lang,
307
            ]
308
        );
309
        $query = $table->find('all')
310
            ->where($conditions)
311
            ->notMatching('Translations', function ($q) use ($to) {
312
                return $q->where(['Translations.lang' => $to]);
313
            })
314
            ->orderAsc($table->aliasField('id'))
315
            ->limit(500);
316
        $lastId = 0;
317
        while (true) {
318
            $q = clone $query;
319
            $q = $q->where(fn (QueryExpression $exp): QueryExpression => $exp->gt($table->aliasField('id'), $lastId));
320
            $results = $q->all();
321
            if ($results->isEmpty()) {
322
                break;
323
            }
324
325
            foreach ($results as $entity) {
326
                $lastId = $entity->id;
327
328
                yield $entity;
329
            }
330
        }
331
    }
332
333
    /**
334
     * Translate object translatable fields and store translation in translations table.
335
     *
336
     * @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate
337
     * @param string $from The language to translate from
338
     * @param string $to The language to translate to
339
     * @return void
340
     */
341
    public function translate($object, $from, $to): void
342
    {
343
        $id = $object->id;
344
        $type = $object->type;
345
        $object = $this->fetchTable($type)->find()->where(compact('id'))->firstOrFail();
346
        $translatableFields = $this->translatableFields($type);
347
        if (empty($translatableFields)) {
348
            return;
349
        }
350
        $translatedFields = [];
351
        $fields = $values = $jsonFields = $jsonValues = [];
352
        foreach ($translatableFields as $field) {
353
            if (empty($object->get($field))) {
354
                continue;
355
            }
356
            $val = $object->get($field);
357
            if (is_array($val)) {
358
                $jsonFields[] = $field;
359
                $jsonValues[] = json_encode($val);
360
            } else {
361
                $fields[] = $field;
362
                $values[] = $val;
363
            }
364
        }
365
        if (empty($fields) && empty($jsonFields)) {
366
            return;
367
        }
368
        $tr = $this->multiTranslation($values, $from, $to);
369
        foreach ($tr as $i => $t) {
370
            $translatedFields[$fields[$i]] = $t;
371
        }
372
        $tr = $this->multiTranslation($jsonValues, $from, $to);
373
        foreach ($tr as $i => $t) {
374
            $translatedFields[$jsonFields[$i]] = json_decode($t, true);
375
        }
376
        $translation = [
377
            'object_id' => $id,
378
            'lang' => array_flip($this->langsMap)[$to],
379
            'translated_fields' => json_encode($translatedFields),
380
            'status' => $this->defaultStatus,
381
        ];
382
        $table = $this->fetchTable('Translations');
383
        $entity = $table->newEntity($translation);
384
        $entity->set('translated_fields', json_decode($translation['translated_fields'], true));
385
        $entity->set('status', $this->defaultStatus);
386
        LoggedUser::setUserAdmin();
387
        $table->saveOrFail($entity);
388
    }
389
390
    /**
391
     * Get translatable fields by object type.
392
     *
393
     * @param string $type The object type
394
     * @return array
395
     */
396
    public function translatableFields(string $type): array
397
    {
398
        if (array_key_exists($type, $this->translatableFields)) {
399
            return $this->translatableFields[$type];
400
        }
401
        $schema = JsonSchema::typeSchema($type);
402
        $this->translatableFields[$type] = (array)Hash::get($schema, 'translatable');
403
404
        return $this->translatableFields[$type];
405
    }
406
407
    /**
408
     * Translate a single text.
409
     *
410
     * @param mixed $text The text to translate
411
     * @param string $from The language to translate from
412
     * @param string $to The language to translate to
413
     * @return string
414
     */
415
    public function singleTranslation($text, string $from, string $to): string
416
    {
417
        $response = $this->translator->translate([$text], $from, $to);
418
        $response = json_decode($response, true);
419
420
        return (string)Hash::get($response, 'translation.0');
421
    }
422
423
    /**
424
     * Translate multiple texts.
425
     *
426
     * @param array $texts The texts to translate
427
     * @param string $from The language to translate from
428
     * @param string $to The language to translate to
429
     * @return array
430
     */
431
    public function multiTranslation($texts, string $from, string $to): array
432
    {
433
        $response = $this->translator->translate($texts, $from, $to);
434
        $response = json_decode($response, true);
435
436
        return (array)Hash::get($response, 'translation');
437
    }
438
}
439