ModelsExtractor   F
last analyzed

Complexity

Total Complexity 101

Size/Duplication

Total Lines 424
Duplicated Lines 0 %

Test Coverage

Coverage 0%

Importance

Changes 4
Bugs 0 Features 1
Metric Value
wmc 101
eloc 245
c 4
b 0
f 1
dl 0
loc 424
rs 2
ccs 0
cts 231
cp 0

10 Methods

Rating   Name   Duplication   Size   Complexity  
A reset() 0 2 1
A deleteOldRecords() 0 13 6
A getAttributesNamesForFind() 0 9 3
A triggerOperation() 0 2 1
F getModels() 0 287 73
C saveModel() 0 43 12
A logError() 0 6 2
A isBadRow() 0 2 1
A startOperation() 0 3 1
A endOperation() 0 3 1

How to fix   Complexity   

Complex Class

Complex classes like ModelsExtractor often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.

Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.

While breaking up the class, it is a good idea to analyze how other classes use ModelsExtractor, and based on these observations, apply Extract Interface, too.

1
<?php
2
/**
3
 * Created by PhpStorm.
4
 * User: execut
5
 * Date: 12/22/17
6
 * Time: 12:57 PM
7
 */
8
9
namespace execut\import\components;
10
11
12
use execut\import\ModelInterface;
13
use execut\import\Query;
14
use yii\base\Component;
15
use yii\base\Exception;
16
use yii\db\ActiveQuery;
17
use yii\db\ActiveRecord;
18
use yii\helpers\ArrayHelper;
19
20
class ModelsExtractor extends Component
21
{
22
    public $id = '';
23
24
    /**
25
     * @var ActiveQuery $query
26
     */
27
    public $query;
28
    public $scopes = null;
29
    public $isImport = false;
30
    public $attributes = [];
31
    public $scenario = null;
32
    /**
33
     * @var Importer
34
     */
35
    public $importer = null;
36
    protected $modelsByUniqueKey = [];
37
    public $isNoCreate = false;
38
    public $isNoUpdate = false;
39
    public $isDelete = false;
40
    public $deletedIds = [];
41
    public $uniqueKeys = null;
42
43
    public function reset() {
44
        $this->modelsByUniqueKey = [];
45
    }
46
47
    public function deleteOldRecords() {
48
        if ($this->isDelete) {
49
            $ids = $this->deletedIds;
50
            if (!empty($ids)) {
51
                if (count($ids) > 500000) {
52
                    throw new Exception('Many than 500 000 records to delete. Dangerous situation.');
53
                }
54
55
                $modelClass = $this->query->modelClass;
56
                while ($idsPart = array_splice($ids, 0, 65534)) {
57
                    if (count($idsPart) > 0) {
58
                        $modelClass::deleteAll([
59
                            'id' => $idsPart
60
                        ]);
61
                    }
62
                }
63
            }
64
        }
65
    }
66
67
    public function getModels($isSave = true, $isMarkBad = true) {
68
        /**
69
         * @var ActiveRecord $model
70
         */
71
        $this->startOperation('extract');
72
        $whereValues = [];
73
        $relationsModels = [];
74
        $this->startOperation('construct where');
75
        foreach ($this->attributes as $attribute => $attributeParams) {
76
            if (empty($attributeParams['extractor'])) {
77
                $extractorId = $attribute;
78
            } else {
79
                $extractorId = $attributeParams['extractor'];
80
            }
81
82
            if (empty($attributeParams['value']) && ($extractor = $this->importer->getExtractor($extractorId))) {
83
                $models = $extractor->getModels(false, $isMarkBad && !empty($attributeParams['isFind']));
84
                foreach ($this->importer->rows as $rowNbr => $row) {
0 ignored issues
show
Bug Best Practice introduced by eXeCUT
The property rows does not exist on execut\import\components\Importer. Since you implemented __get, consider adding a @property annotation.
Loading history...
85
                    if ($this->isBadRow($rowNbr)) {
86
                        continue;
87
                    }
88
89
                    if (empty($models[$rowNbr])) {
90
                        if ($isMarkBad && !empty($attributeParams['isFind'])) {
91
                            $this->importer->setIsBadRow($rowNbr);
92
                        }
93
94
                        continue;
95
                    }
96
97
                    $model = $models[$rowNbr];
98
99
                    if (empty($relationsModels[$rowNbr])) {
100
                        $relationsModels[$rowNbr] = [];
101
                    }
102
103
                    if (empty($extractor->isNoCreate) || !$model->isNewRecord) {
104
                        $relationsModels[$rowNbr][$attribute] = $model;
105
                    }
106
107
                    if (!empty($attributeParams['isFind'])) {
108
                        if (!$model->isNewRecord && !empty($model->dirtyAttributes)) {
109
                            if (empty($extractor->isNoUpdate)) {
110
                                if (!$this->saveModel($model, $rowNbr)) {
111
                                    unset($whereValues[$rowNbr]);
112
                                    continue;
113
                                }
114
                            }
115
                        } else if ($model->isNewRecord) {
116
                            if (empty($extractor->isNoCreate)) {
117
                                // Поиск не нужен, модель новая
118
                                unset($whereValues[$rowNbr]);
119
                                $this->saveModel($model, $rowNbr);
120
                                continue;
121
                            }
122
                        }
123
124
                        if ($model->isNewRecord && !empty($extractor->isNoCreate)) {
125
                            $this->logError('Related record is not founded with attributes ' . serialize(array_filter($model->attributes)), $rowNbr, $model, null, $isMarkBad);
126
                            unset($whereValues[$rowNbr]);
127
                            continue;
128
                        }
129
130
                        if (empty($whereValues[$rowNbr])) {
131
                            $whereValues[$rowNbr] = [];
132
                        }
133
134
                        $whereValues[$rowNbr][$attribute] = (int)$model->id;
135
                    } else {
136
                        if ($model->isNewRecord && empty($extractor->isNoCreate)) {
137
                            if (!$this->saveModel($model, $rowNbr, false)) {
138
                                unset($relationsModels[$rowNbr][$attribute]);
139
                            }
140
                        }
141
                    }
142
                }
143
            } else {
144
                if (!empty($attributeParams['isFind'])) {
145
                    foreach ($this->importer->rows as $rowNbr => $row) {
146
                        if ($this->isBadRow($rowNbr)) {
147
                            continue;
148
                        }
149
150
                        if (!empty($attributeParams['value'])) {
151
                            $whereValues[$rowNbr][$attribute] = $attributeParams['value'];
152
                            continue;
153
                        }
154
155
                        if (empty($attributeParams['column'])) {
156
                            return [];
157
                            throw new Exception('Column key is required for attribute params. Extractor: ' . $extractorId . '. Attribute params: ' . var_export($attributeParams, true));
0 ignored issues
show
Unused Code introduced by eXeCUT
ThrowNode is not reachable.

This check looks for unreachable code. It uses sophisticated control flow analysis techniques to find statements which will never be executed.

Unreachable code is most often the result of return, die or exit statements that have been added for debug purposes.

function fx() {
    try {
        doSomething();
        return true;
    }
    catch (\Exception $e) {
        return false;
    }

    return false;
}

In the above example, the last return false will never be executed, because a return statement has already been met in every possible execution path.

Loading history...
158
                        }
159
160
                        if (empty($row[$attributeParams['column'] - 1])) {
161
                            $this->logError($attribute . ' is required for record find. Extractor: ' . $this->id, $rowNbr, null, $attributeParams['column'], $isMarkBad);
162
                            unset($whereValues[$rowNbr]);
163
                            continue;
164
                        }
165
166
                        if (empty($whereValues[$rowNbr])) {
167
                            $whereValues[$rowNbr] = [];
168
                        }
169
170
                        $modelClass = $this->query->modelClass;
171
                        $value = $row[$attributeParams['column'] - 1];
172
                        if (method_exists($modelClass, 'filtrateAttribute')) {
173
                            $value = $modelClass::filtrateAttribute($attribute, $value);
174
                        }
175
176
                        $whereValues[$rowNbr][$attribute] = $value;
177
                    }
178
                }
179
            }
180
        }
181
182
        $result = [];
183
        foreach ($whereValues as $rowNbr => $whereValue) {
184
            if ($this->importer->isBadRow($rowNbr)) {
185
                continue;
186
            }
187
188
            $uniqueKey = serialize($whereValue);
189
            if (!isset($this->modelsByUniqueKey[$uniqueKey])) {
190
                $result[$uniqueKey] = $whereValue;
191
            }
192
        }
193
194
        $whereValues = $result;
195
196
        $this->endOperation('construct where');
197
        if (count($whereValues)) {
198
            $attributesNames = $this->getAttributesNamesForFind();
199
            $query = clone $this->query;
200
            $query->indexBy(function ($row) use ($attributesNames) {
201
                $searchedAttributes = [];
202
                foreach ($attributesNames as $attributesName) {
203
                    $modelClass = $this->query->modelClass;
204
                    $value = $row[$attributesName];
205
                    if (method_exists($modelClass, 'filtrateAttribute')) {
206
                        $value = $modelClass::filtrateAttribute($attributesName, $value);
207
                    }
208
209
                    $searchedAttributes[$attributesName] = $value;
210
                }
211
                return serialize($searchedAttributes);
212
            });
213
214
            if ($this->scopes !== null) {
215
                foreach ($this->scopes as $scope) {
216
                    $scope($query, [
217
                        'IN',
218
                        $attributesNames,
219
                        $whereValues,
220
                    ]);
221
                }
222
            } else {
223
                $query->andWhere([
224
                    'IN',
225
                    $attributesNames,
226
                    $whereValues,
227
                ]);
228
            }
229
230
            $this->startOperation('find');
231
            $models = $query->all();
232
            $this->endOperation('find');
233
            $this->startOperation('keys collect');
234
            foreach ($models as $uniqueKey => $model) {
235
                if ($this->uniqueKeys !== null) {
236
                    $callback = $this->uniqueKeys;
237
                    $uniqueKeys = $callback($model, $attributesNames, $whereValues);
238
                } else {
239
                    $uniqueKeys = [$uniqueKey];
240
                }
241
242
                foreach ($uniqueKeys as $uniqueKey) {
0 ignored issues
show
Comprehensibility Bug introduced by eXeCUT
$uniqueKey is overwriting a variable from outer foreach loop.
Loading history...
243
                    $this->modelsByUniqueKey[$uniqueKey] = $model;
244
                }
245
            }
246
247
            $this->endOperation('keys collect');
248
        }
249
250
        $models = [];
251
        $this->startOperation('models collect');
252
253
        foreach ($this->importer->rows as $rowNbr => $row) {
254
            if ($this->isBadRow($rowNbr)) {
255
                continue;
256
            }
257
258
            $attributes = [];
259
            foreach ($this->attributes as $attribute => $attributeParams) {
260
                if (empty($attributeParams['extractor'])) {
261
                    $extractorId = $attribute;
262
                } else {
263
                    $extractorId = $attributeParams['extractor'];
264
                }
265
266
                if (empty($attributeParams['value']) && $this->importer->hasExtractor($extractorId)) {
267
                    if (empty($relationsModels[$rowNbr]) || empty($relationsModels[$rowNbr][$attribute])) {
268
//                        $attributes[$attribute] = null;
269
                    } else {
270
                        $p = $relationsModels[$rowNbr][$attribute]->id;
271
                        $attributes[$attribute] = (int)$p;
272
                    }
273
                } else {
274
                    if (!empty($attributeParams['value'])) {
275
                        $attributes[$attribute] = $attributeParams['value'];
276
                    } else {
277
                        if (empty($attributeParams['column'])) {
278
                            throw new Exception('Not setted column for attribute ' . $attribute . ' for extractor ' . $this->id);
279
                        }
280
281
                        $columnNbr = $attributeParams['column'] - 1;
282
                        if (!array_key_exists($columnNbr, $row) || trim($row[$columnNbr]) === '') {
283
                            continue;
284
                        } else {
285
                            $attributes[$attribute] = $row[$columnNbr];
286
                        }
287
                    }
288
                }
289
            }
290
291
            $attributesNames = $this->getAttributesNamesForFind();
292
            $searchedAttributes = [];
293
            foreach ($attributesNames as $attributesName) {
294
                $modelClass = $this->query->modelClass;
295
                if (empty($attributes[$attributesName])) {
296
                    $value = null;
297
                } else {
298
                    $value = $attributes[$attributesName];
299
                }
300
301
                if (method_exists($modelClass, 'filtrateAttribute')) {
302
                    $value = $modelClass::filtrateAttribute($attributesName, $value);
303
                    $attributes[$attributesName] = $value;
304
                }
305
306
                $searchedAttributes[$attributesName] = $value;
307
            }
308
309
            foreach ($attributes as $attributesName => $value) {
310
                if (method_exists($modelClass, 'filtrateAttribute')) {
0 ignored issues
show
Comprehensibility Best Practice introduced by eXeCUT notebook
The variable $modelClass does not seem to be defined for all execution paths leading up to this point.
Loading history...
311
                    $value = $modelClass::filtrateAttribute($attributesName, $value);
312
                    $attributes[$attributesName] = $value;
313
                }
314
            }
315
316
            $uniqueKey = serialize($searchedAttributes);
317
            if (isset($this->modelsByUniqueKey[$uniqueKey])) {
318
                $model = $this->modelsByUniqueKey[$uniqueKey];
319
            } else {
320
                $model = new $this->query->modelClass;
321
                $this->modelsByUniqueKey[$uniqueKey] = $model;
322
            }
323
324
            if ($this->isDelete && !$model->isNewRecord) {
325
                unset($this->deletedIds[$model->id]);
326
            }
327
328
            $modelAttributes = $model->getAttributes(array_keys($attributes));
329
            if (!$isSave || $model->isNewRecord || array_diff($attributes, $modelAttributes)) {
330
                if ($this->scenario) {
331
                    $model->scenario = $this->scenario;
332
                }
333
334
                $model->attributes = $attributes;
335
336
                $models[$rowNbr] = $model;
337
            }
338
        }
339
340
341
342
        $this->endOperation('models collect');
343
344
        $this->endOperation('extract');
345
        if ($isSave) {
346
            foreach ($models as $rowNbr => $model) {
347
                if (!$this->isBadRow($rowNbr)) {
348
                    $this->saveModel($model, $rowNbr);
349
                }
350
            }
351
        }
352
353
        return $models;
354
    }
355
356
    protected $times = [];
357
    protected function startOperation($name) {
358
        echo 'start ' . $name . ' ' . $this->id . "\n";
359
        $this->times[$name] = microtime(true);
360
    }
361
362
    protected function endOperation($name) {
363
        $time = microtime(true) - $this->times[$name];
364
        echo 'end ' . $name . ' ' . $this->id . ' after ' . $time . ' seconds' . "\n";
365
    }
366
367
    protected function triggerOperation($name) {
368
        echo $name . ' ' . $this->id . "\n";
369
    }
370
371
    protected function isBadRow($rowNbr) {
372
        return $this->importer->isBadRow($rowNbr);
373
    }
374
375
    protected function logError($message, $rowNbr, $model, $columnNbr = null, $isMarkBad = true) {
376
        if ($isMarkBad) {
377
            $this->importer->setIsBadRow($rowNbr);
378
        }
379
380
        $this->importer->logError($message, $rowNbr, $model, $columnNbr);
381
    }
382
383
    /**
384
     * @return array
385
     */
386
    public function getAttributesNamesForFind(): array
387
    {
388
        $attributesNames = [];
389
        foreach ($this->attributes as $attribute => $attributeParams) {
390
            if (!empty($attributeParams['isFind'])) {
391
                $attributesNames[] = $attribute;
392
            }
393
        }
394
        return $attributesNames;
395
    }
396
397
    /**
398
     * @param $model
399
     * @return mixed
400
     */
401
    protected function saveModel($model, $rowNbr, $isMarkBad = true)
402
    {
403
        $currentRowNbr = $this->importer->getCurrentStackRowNbr() + $rowNbr;
404
        $modelString = $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes));
405
        echo 'Row #' . $currentRowNbr . ': ';
406
        if ($this->scenario) {
407
            $model->scenario = $this->scenario;
408
        }
409
        if (!$model->validate()) {
410
            $message = 'Error validate ' . $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)) . ' ' . serialize($model->errors);
411
            $this->logError($message, $rowNbr, $model, null, $isMarkBad);
412
            return false;
413
        }
414
415
        $isNewRecord = ($model->isNewRecord && !$this->isNoCreate);
416
        $isUpdatedRecord = (!$model->isNewRecord && !$this->isNoUpdate && !empty($model->dirtyAttributes));
417
        if ($isNewRecord || $isUpdatedRecord) {
418
            if ($isNewRecord) {
419
                $reason = 'created';
420
            } else {
421
                $reason = 'updated';
422
            }
423
424
            echo 'Saving ' . $modelString . ' because they is ' . $reason . "\n";
425
            if ($isUpdatedRecord) {
426
                echo 'Changed attributes ' . serialize(array_keys($model->dirtyAttributes)) . "\n";
427
                $oldValues = [];
428
                foreach ($model->dirtyAttributes as $attribute => $value) {
429
                    $oldValues[$attribute] = $model->oldAttributes[$attribute];
430
                }
431
432
                echo 'Old values ' . serialize($oldValues) . "\n";
433
                echo 'New values ' . serialize($model->dirtyAttributes) . "\n";
434
            }
435
436
            if (!$model->save()) {
437
                $this->logError('Error while saving ' . $modelString, $rowNbr, $model);
438
            }
439
        } else {
440
            echo $modelString . ' Is skipped because is not changed' . "\n";
441
        }
442
443
        return true;
444
    }
445
}