Passed
Push — main ( d47134...ca2b40 )
by Dante
12:54
created

TranslateObjectsCommand::setIo()   A

Complexity

Conditions 1
Paths 1

Size

Total Lines 3
Code Lines 1

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 0 Features 0
Metric Value
eloc 1
c 1
b 0
f 0
dl 0
loc 3
rs 10
cc 1
nc 1
nop 1
1
<?php
2
declare(strict_types=1);
3
4
/**
5
 * BEdita, API-first content management framework
6
 * Copyright 2024 Atlas Srl, Chialab Srl
7
 *
8
 * This file is part of BEdita: you can redistribute it and/or modify
9
 * it under the terms of the GNU Lesser General Public License as published
10
 * by the Free Software Foundation, either version 3 of the License, or
11
 * (at your option) any later version.
12
 *
13
 * See LICENSE.LGPL or <http://gnu.org/licenses/lgpl-3.0.html> for more details.
14
 */
15
namespace BEdita\ImportTools\Command;
16
17
use BEdita\Core\Model\Entity\ObjectEntity;
18
use BEdita\Core\Utility\JsonSchema;
19
use BEdita\Core\Utility\LoggedUser;
20
use Cake\Command\Command;
21
use Cake\Console\Arguments;
22
use Cake\Console\ConsoleIo;
23
use Cake\Console\ConsoleOptionParser;
24
use Cake\Core\Configure;
25
use Cake\Core\InstanceConfigTrait;
26
use Cake\Database\Expression\QueryExpression;
27
use Cake\Utility\Hash;
28
29
/**
30
 * TranslateObjects command.
31
 * Translate objects from a language to another using a translator engine.
32
 * The translator engine is defined in the configuration.
33
 * The configuration must contain the translator engine class and options
34
 * I.e.:
35
 * 'Translators' => [
36
 *   'deepl' => [
37
 *      'name' => 'DeepL',
38
 *      'class' => '\BEdita\I18n\Deepl\Core\Translator',
39
 *      'options' => [
40
 *        'auth_key' => '************',
41
 *      ],
42
 *   ],
43
 * ],
44
 */
45
class TranslateObjectsCommand extends Command
46
{
47
    use InstanceConfigTrait;
48
49
    protected $_defaultConfig = [
50
        'langsMap' => [
51
            'en' => 'en-US',
52
            'it' => 'it',
53
            'de' => 'de',
54
            'es' => 'es',
55
            'fr' => 'fr',
56
            'pt' => 'pt-PT',
57
        ],
58
        'status' => 'draft',
59
        'dryRun' => false,
60
    ];
61
    protected $ok;
62
    protected $error;
63
    protected $io;
64
    protected $dryRun;
65
    protected $defaultStatus;
66
    protected $translatableFields = [];
67
    protected $translator;
68
    protected $langsMap;
69
    protected $limit;
70
    protected $type;
71
72
    /**
73
     * @inheritDoc
74
     */
75
    public function __construct()
76
    {
77
        parent::__construct();
78
        $cfg = (array)Configure::read('TranslateObjects');
79
        $cfg = array_merge($this->_defaultConfig, $cfg);
80
        $this->defaultStatus = (string)Hash::get($cfg, 'status');
81
        $this->setDryRun(Hash::get($cfg, 'dryRun') === 1);
82
        $this->langsMap = (array)Hash::get($cfg, 'langsMap');
83
    }
84
85
    /**
86
     * @inheritDoc
87
     */
88
    public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser
89
    {
90
        $langs = array_keys($this->langsMap);
91
        $parser = parent::buildOptionParser($parser);
92
        $parser->addOption('from', [
93
            'short' => 'f',
94
            'help' => 'Language to translate from',
95
            'required' => true,
96
            'choices' => $langs,
97
        ]);
98
        $parser->addOption('to', [
99
            'short' => 't',
100
            'help' => 'Language to translate to',
101
            'required' => true,
102
            'choices' => $langs,
103
        ]);
104
        $parser->addOption('engine', [
105
            'short' => 'e',
106
            'help' => 'Translator engine',
107
            'default' => 'deepl',
108
            'required' => true,
109
            'choices' => ['aws', 'deepl', 'google', 'microsoft'],
110
        ]);
111
        $parser->addOption('status', [
112
            'short' => 's',
113
            'help' => 'Status for new translations',
114
            'choices' => ['draft', 'on', 'off'],
115
            'default' => 'draft',
116
        ]);
117
        $parser->addOption('dry-run', [
118
            'short' => 'd',
119
            'help' => 'Dry run',
120
            'boolean' => true,
121
            'default' => false,
122
        ]);
123
        $parser->addOption('limit', [
124
            'short' => 'l',
125
            'help' => 'Limit number of objects to translate',
126
        ]);
127
        $parser->addOption('object-type', [
128
            'short' => 'o',
129
            'help' => 'Object type to translate',
130
        ]);
131
132
        return $parser;
133
    }
134
135
    /**
136
     * @inheritDoc
137
     */
138
    public function execute(Arguments $args, ConsoleIo $io): void
139
    {
140
        $this->setDryRun($args->getOption('dry-run'));
141
        $this->setIo($io);
142
143
        // parameter engine
144
        $engine = $args->getOption('engine') ?? 'deepl';
145
146
        // setup translator engine
147
        $cfg = Configure::read(sprintf('Translators.%s', $engine));
148
        if (empty($cfg)) {
149
            $this->getIo()->abort(sprintf('Translator %s not found', $engine));
150
        }
151
        $this->setTranslator($cfg);
152
153
        // parameters: lang from, lang to
154
        $from = $args->getOption('from');
155
        $to = $args->getOption('to');
156
        $this->limit = $args->getOption('limit') ?? null;
157
        $this->type = $args->getOption('object-type') ?? null;
158
        $this->getIo()->out(
159
            sprintf(
160
                'Translating objects from %s to %s [dry-run %s / limit %s / %s]',
161
                $from,
162
                $to,
163
                $this->dryRun ? 'yes' : 'no',
164
                $this->limit ? sprintf('limit %s', $this->limit) : 'unlimited',
165
                $this->type ? sprintf('type %s', $this->type) : 'all types'
166
            )
167
        );
168
        $to = $this->langsMap[$to];
169
        if ($this->getIo()->ask('Do you want to continue [Y/n]?', 'n') !== 'Y') {
170
            $this->getIo()->abort('Bye');
171
        }
172
        $this->ok = $this->error = 0;
173
        $this->defaultStatus = $args->getOption('status');
174
        $this->processObjects($from, $to);
175
        $this->getIo()->out($this->results());
176
        $this->getIo()->out('Done');
177
    }
178
179
    /**
180
     * Set console io.
181
     *
182
     * @param \Cake\Console\ConsoleIo $io The console io
183
     * @return void
184
     */
185
    public function setIo(ConsoleIo $io): void
186
    {
187
        $this->io = $io;
188
    }
189
190
    /**
191
     * Get console io.
192
     *
193
     * @return \Cake\Console\ConsoleIo
194
     */
195
    public function getIo(): ConsoleIo
196
    {
197
        return $this->io;
198
    }
199
200
    /**
201
     * Set dry run.
202
     *
203
     * @param bool $dryRun The dry run flag
204
     * @return void
205
     */
206
    public function setDryRun(bool $dryRun): void
207
    {
208
        $this->dryRun = $dryRun;
209
    }
210
211
    /**
212
     * Get dry run.
213
     *
214
     * @return bool
215
     */
216
    public function getDryRun(): bool
217
    {
218
        return $this->dryRun;
219
    }
220
221
    /**
222
     * Get results.
223
     *
224
     * @return string
225
     */
226
    public function results(): string
227
    {
228
        return sprintf('Processed %d objects (%d errors)', $this->ok + $this->error, $this->error);
229
    }
230
231
    /**
232
     * Set translator engine.
233
     *
234
     * @param array $cfg The translator configuration
235
     * @return void
236
     */
237
    public function setTranslator(array $cfg): void
238
    {
239
        $class = (string)Hash::get($cfg, 'class');
240
        $options = (array)Hash::get($cfg, 'options');
241
        $this->translator = new $class();
242
        $this->translator->setup($options);
243
    }
244
245
    /**
246
     * Process objects to translate.
247
     *
248
     * @param string $from The language to translate from
249
     * @param string $to The language to translate to
250
     * @return void
251
     */
252
    public function processObjects(string $from, string $to): void
253
    {
254
        $conditions = [];
255
        foreach ($this->objectsIterator($conditions, $from, $to) as $object) {
256
            if ($this->limit !== null && ($this->ok + $this->error >= $this->limit)) {
257
                break;
258
            }
259
            $this->processObject($object, $from, $to);
260
        }
261
    }
262
263
    /**
264
     * Process single object.
265
     *
266
     * @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate
267
     * @param string $from The language to translate from
268
     * @param string $to The language to translate to
269
     * @return void
270
     */
271
    public function processObject(ObjectEntity $object, string $from, string $to): void
272
    {
273
        try {
274
            $this->getIo()->verbose(sprintf('Translating object %s', $object->id));
275
            if (!$this->dryRun) {
276
                $this->translate($object, $from, $to);
277
            }
278
            $this->getIo()->verbose(sprintf('Translated object %s', $object->id));
279
            $this->ok++;
280
        } catch (\Exception $e) {
281
            $this->getIo()->error(sprintf('Error translating object %s: %s', $object->id, $e->getMessage()));
282
            $this->error++;
283
        }
284
    }
285
286
    /**
287
     * Get objects as iterable.
288
     *
289
     * @param array $conditions The conditions to filter objects.
290
     * @param string $lang The language to use to find objects.
291
     * @param string $to The language to translate objects to.
292
     * @return iterable
293
     */
294
    public function objectsIterator(array $conditions, string $lang, string $to): iterable
295
    {
296
        /** @var \BEdita\Core\Model\Table\ObjectsTable $table */
297
        $table = $this->fetchTable('objects');
298
        if ($this->type !== null) {
299
            $conditions[$table->aliasField('object_type_id')] = $table->objectType($this->type)->id;
300
        }
301
        $conditions = array_merge(
302
            $conditions,
303
            [
304
                $table->aliasField('deleted') => 0,
305
                $table->aliasField('lang') => $lang,
306
            ]
307
        );
308
        $query = $table->find('all')
309
            ->where($conditions)
310
            ->notMatching('Translations', function ($q) use ($to) {
311
                return $q->where(['Translations.lang' => $to]);
312
            })
313
            ->orderAsc($table->aliasField('id'))
314
            ->limit(500);
315
        $lastId = 0;
316
        while (true) {
317
            $q = clone $query;
318
            $q = $q->where(fn (QueryExpression $exp): QueryExpression => $exp->gt($table->aliasField('id'), $lastId));
319
            $results = $q->all();
320
            if ($results->isEmpty()) {
321
                break;
322
            }
323
324
            foreach ($results as $entity) {
325
                $lastId = $entity->id;
326
327
                yield $entity;
328
            }
329
        }
330
    }
331
332
    /**
333
     * Translate object translatable fields and store translation in translations table.
334
     *
335
     * @param \BEdita\Core\Model\Entity\ObjectEntity $object The object to translate
336
     * @param string $from The language to translate from
337
     * @param string $to The language to translate to
338
     * @return void
339
     */
340
    public function translate($object, $from, $to): void
341
    {
342
        $id = $object->id;
343
        $type = $object->type;
344
        $object = $this->fetchTable($type)->find()->where(compact('id'))->firstOrFail();
345
        $translatableFields = $this->translatableFields($type);
346
        if (empty($translatableFields)) {
347
            return;
348
        }
349
        $translatedFields = [];
350
        $fields = $values = $jsonFields = $jsonValues = [];
351
        foreach ($translatableFields as $field) {
352
            if (empty($object->get($field))) {
353
                continue;
354
            }
355
            $val = $object->get($field);
356
            if (is_array($val)) {
357
                $jsonFields[] = $field;
358
                $jsonValues[] = json_encode($val);
359
            } else {
360
                $fields[] = $field;
361
                $values[] = $val;
362
            }
363
        }
364
        if (empty($fields) && empty($jsonFields)) {
365
            return;
366
        }
367
        $tr = $this->multiTranslation($values, $from, $to);
368
        foreach ($tr as $i => $t) {
369
            $translatedFields[$fields[$i]] = $t;
370
        }
371
        $tr = $this->multiTranslation($jsonValues, $from, $to);
372
        foreach ($tr as $i => $t) {
373
            $translatedFields[$jsonFields[$i]] = json_decode($t, true);
374
        }
375
        $translation = [
376
            'object_id' => $id,
377
            'lang' => array_flip($this->langsMap)[$to],
378
            'translated_fields' => json_encode($translatedFields),
379
            'status' => $this->defaultStatus,
380
        ];
381
        $table = $this->fetchTable('Translations');
382
        $entity = $table->newEntity($translation);
383
        $entity->set('translated_fields', json_decode($translation['translated_fields'], true));
384
        $entity->set('status', $this->defaultStatus);
385
        LoggedUser::setUserAdmin();
386
        $table->saveOrFail($entity);
387
    }
388
389
    /**
390
     * Get translatable fields by object type.
391
     *
392
     * @param string $type The object type
393
     * @return array
394
     */
395
    public function translatableFields(string $type): array
396
    {
397
        if (array_key_exists($type, $this->translatableFields)) {
398
            return $this->translatableFields[$type];
399
        }
400
        $schema = JsonSchema::typeSchema($type);
401
        $this->translatableFields[$type] = (array)Hash::get($schema, 'translatable');
402
403
        return $this->translatableFields[$type];
404
    }
405
406
    /**
407
     * Translate a single text.
408
     *
409
     * @param mixed $text The text to translate
410
     * @param string $from The language to translate from
411
     * @param string $to The language to translate to
412
     * @return string
413
     */
414
    public function singleTranslation($text, string $from, string $to): string
415
    {
416
        $response = $this->translator->translate([$text], $from, $to);
417
        $response = json_decode($response, true);
418
419
        return (string)Hash::get($response, 'translation.0');
420
    }
421
422
    /**
423
     * Translate multiple texts.
424
     *
425
     * @param array $texts The texts to translate
426
     * @param string $from The language to translate from
427
     * @param string $to The language to translate to
428
     * @return array
429
     */
430
    public function multiTranslation($texts, string $from, string $to): array
431
    {
432
        $response = $this->translator->translate($texts, $from, $to);
433
        $response = json_decode($response, true);
434
435
        return (array)Hash::get($response, 'translation');
436
    }
437
}
438