ModelsExtractor   F
last analyzed

Complexity

Total Complexity 96

Size/Duplication

Total Lines 410
Duplicated Lines 0 %

Test Coverage

Coverage 0%

Importance

Changes 2
Bugs 0 Features 0
Metric Value
wmc 96
eloc 238
c 2
b 0
f 0
dl 0
loc 410
ccs 0
cts 226
cp 0
rs 2

10 Methods

Rating   Name   Duplication   Size   Complexity  
A reset() 0 2 1
A deleteOldRecords() 0 13 6
A getAttributesNamesForFind() 0 9 3
A triggerOperation() 0 2 1
F getModels() 0 276 69
B saveModel() 0 41 11
A logError() 0 6 2
A isBadRow() 0 2 1
A startOperation() 0 3 1
A endOperation() 0 3 1

How to fix   Complexity   

Complex Class

Complex classes like ModelsExtractor often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.

Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.

While breaking up the class, it is a good idea to analyze how other classes use ModelsExtractor, and based on these observations, apply Extract Interface, too.

1
<?php
2
/**
3
 * Created by PhpStorm.
4
 * User: execut
5
 * Date: 12/22/17
6
 * Time: 12:57 PM
7
 */
8
9
namespace execut\import\components;
10
11
12
use execut\import\ModelInterface;
13
use execut\import\Query;
14
use yii\base\Component;
15
use yii\base\Exception;
16
use yii\db\ActiveQuery;
17
use yii\db\ActiveRecord;
18
use yii\helpers\ArrayHelper;
19
20
class ModelsExtractor extends Component
21
{
22
    public $id = '';
23
24
    /**
25
     * @var ActiveQuery $query
26
     */
27
    public $query;
28
    public $scopes = null;
29
    public $isImport = false;
30
    public $attributes = [];
31
    /**
32
     * @var Importer
33
     */
34
    public $importer = null;
35
    protected $modelsByUniqueKey = [];
36
    public $isNoCreate = false;
37
    public $isNoUpdate = false;
38
    public $isDelete = false;
39
    public $deletedIds = [];
40
    public $uniqueKeys = null;
41
42
    public function reset() {
43
        $this->modelsByUniqueKey = [];
44
    }
45
46
    public function deleteOldRecords() {
47
        if ($this->isDelete) {
48
            $ids = $this->deletedIds;
49
            if (!empty($ids)) {
50
                if (count($ids) > 500000) {
51
                    throw new Exception('Many than 500 000 records to delete. Dangerous situation.');
52
                }
53
54
                $modelClass = $this->query->modelClass;
55
                while ($idsPart = array_splice($ids, 0, 65534)) {
56
                    if (count($idsPart) > 0) {
57
                        $modelClass::deleteAll([
58
                            'id' => $idsPart
59
                        ]);
60
                    }
61
                }
62
            }
63
        }
64
    }
65
66
    public function getModels($isSave = true, $isMarkBad = true) {
67
        /**
68
         * @var ActiveRecord $model
69
         */
70
        $this->startOperation('extract');
71
        $whereValues = [];
72
        $relationsModels = [];
73
        $this->startOperation('construct where');
74
        foreach ($this->attributes as $attribute => $attributeParams) {
75
            if (empty($attributeParams['extractor'])) {
76
                $extractorId = $attribute;
77
            } else {
78
                $extractorId = $attributeParams['extractor'];
79
            }
80
81
            if (empty($attributeParams['value']) && ($extractor = $this->importer->getExtractor($extractorId))) {
82
                $models = $extractor->getModels(false, $isMarkBad && !empty($attributeParams['isFind']));
83
                foreach ($this->importer->rows as $rowNbr => $row) {
0 ignored issues
show
Bug Best Practice introduced by eXeCUT
The property rows does not exist on execut\import\components\Importer. Since you implemented __get, consider adding a @property annotation.
Loading history...
84
                    if ($this->isBadRow($rowNbr)) {
85
                        continue;
86
                    }
87
88
                    if (empty($models[$rowNbr])) {
89
                        if ($isMarkBad && !empty($attributeParams['isFind'])) {
90
                            $this->importer->setIsBadRow($rowNbr);
91
                        }
92
93
                        continue;
94
                    }
95
96
                    $model = $models[$rowNbr];
97
98
                    if (empty($relationsModels[$rowNbr])) {
99
                        $relationsModels[$rowNbr] = [];
100
                    }
101
102
                    if (empty($extractor->isNoCreate) || !$model->isNewRecord) {
103
                        $relationsModels[$rowNbr][$attribute] = $model;
104
                    }
105
106
                    if (!empty($attributeParams['isFind'])) {
107
                        if (!$model->isNewRecord && !empty($model->dirtyAttributes)) {
108
                            if (empty($extractor->isNoUpdate)) {
109
                                if (!$this->saveModel($model, $rowNbr)) {
110
                                    unset($whereValues[$rowNbr]);
111
                                    continue;
112
                                }
113
                            }
114
                        } else if ($model->isNewRecord) {
115
                            if (empty($extractor->isNoCreate)) {
116
                                // Поиск не нужен, модель новая
117
                                unset($whereValues[$rowNbr]);
118
                                $this->saveModel($model, $rowNbr);
119
                                continue;
120
                            }
121
                        }
122
123
                        if ($model->isNewRecord && !empty($extractor->isNoCreate)) {
124
                            $this->logError('Related record is not founded with attributes ' . serialize(array_filter($model->attributes)), $rowNbr, $model, null, $isMarkBad);
125
                            unset($whereValues[$rowNbr]);
126
                            continue;
127
                        }
128
129
                        if (empty($whereValues[$rowNbr])) {
130
                            $whereValues[$rowNbr] = [];
131
                        }
132
133
                        $whereValues[$rowNbr][$attribute] = (int)$model->id;
134
                    }
135
                }
136
            } else {
137
                if (!empty($attributeParams['isFind'])) {
138
                    foreach ($this->importer->rows as $rowNbr => $row) {
139
                        if ($this->isBadRow($rowNbr)) {
140
                            continue;
141
                        }
142
143
                        if (!empty($attributeParams['value'])) {
144
                            $whereValues[$rowNbr][$attribute] = $attributeParams['value'];
145
                            continue;
146
                        }
147
148
                        if (empty($attributeParams['column'])) {
149
                            return [];
150
                            throw new Exception('Column key is required for attribute params. Extractor: ' . $extractorId . '. Attribute params: ' . var_export($attributeParams, true));
0 ignored issues
show
Unused Code introduced by eXeCUT
ThrowNode is not reachable.

This check looks for unreachable code. It uses sophisticated control flow analysis techniques to find statements which will never be executed.

Unreachable code is most often the result of return, die or exit statements that have been added for debug purposes.

function fx() {
    try {
        doSomething();
        return true;
    }
    catch (\Exception $e) {
        return false;
    }

    return false;
}

In the above example, the last return false will never be executed, because a return statement has already been met in every possible execution path.

Loading history...
151
                        }
152
153
                        if (empty($row[$attributeParams['column'] - 1])) {
154
                            $this->logError($attribute . ' is required for record find', $rowNbr, null, $attributeParams['column']);
155
                            unset($whereValues[$rowNbr]);
156
                            continue;
157
                        }
158
159
                        if (empty($whereValues[$rowNbr])) {
160
                            $whereValues[$rowNbr] = [];
161
                        }
162
163
                        $modelClass = $this->query->modelClass;
164
                        $value = $row[$attributeParams['column'] - 1];
165
                        if (method_exists($modelClass, 'filtrateAttribute')) {
166
                            $value = $modelClass::filtrateAttribute($attribute, $value);
167
                        }
168
169
                        $whereValues[$rowNbr][$attribute] = $value;
170
                    }
171
                }
172
            }
173
        }
174
175
        $result = [];
176
        foreach ($whereValues as $rowNbr => $whereValue) {
177
            if ($this->importer->isBadRow($rowNbr)) {
178
                continue;
179
            }
180
181
            $uniqueKey = serialize($whereValue);
182
            if (!isset($this->modelsByUniqueKey[$uniqueKey])) {
183
                $result[$uniqueKey] = $whereValue;
184
            }
185
        }
186
187
        $whereValues = $result;
188
189
        $this->endOperation('construct where');
190
        if (count($whereValues)) {
191
            $attributesNames = $this->getAttributesNamesForFind();
192
            $query = clone $this->query;
193
            $query->indexBy(function ($row) use ($attributesNames) {
194
                $searchedAttributes = [];
195
                foreach ($attributesNames as $attributesName) {
196
                    $modelClass = $this->query->modelClass;
197
                    $value = $row[$attributesName];
198
                    if (method_exists($modelClass, 'filtrateAttribute')) {
199
                        $value = $modelClass::filtrateAttribute($attributesName, $value);
200
                    }
201
202
                    $searchedAttributes[$attributesName] = $value;
203
                }
204
                return serialize($searchedAttributes);
205
            });
206
207
            if ($this->scopes !== null) {
208
                foreach ($this->scopes as $scope) {
209
                    $scope($query, [
210
                        'IN',
211
                        $attributesNames,
212
                        $whereValues,
213
                    ]);
214
                }
215
            } else {
216
                $query->andWhere([
217
                    'IN',
218
                    $attributesNames,
219
                    $whereValues,
220
                ]);
221
            }
222
223
            $this->startOperation('find');
224
            $models = $query->all();
225
            $this->endOperation('find');
226
            $this->startOperation('keys collect');
227
            foreach ($models as $uniqueKey => $model) {
228
                if ($this->uniqueKeys !== null) {
229
                    $callback = $this->uniqueKeys;
230
                    $uniqueKeys = $callback($model, $attributesNames, $whereValues);
231
                } else {
232
                    $uniqueKeys = [$uniqueKey];
233
                }
234
235
                foreach ($uniqueKeys as $uniqueKey) {
0 ignored issues
show
Comprehensibility Bug introduced by eXeCUT
$uniqueKey is overwriting a variable from outer foreach loop.
Loading history...
236
                    $this->modelsByUniqueKey[$uniqueKey] = $model;
237
                }
238
            }
239
240
            $this->endOperation('keys collect');
241
        }
242
243
        $models = [];
244
        $this->startOperation('models collect');
245
        foreach ($this->importer->rows as $rowNbr => $row) {
246
            if ($this->isBadRow($rowNbr)) {
247
                continue;
248
            }
249
250
            $attributes = [];
251
            foreach ($this->attributes as $attribute => $attributeParams) {
252
                if (empty($attributeParams['extractor'])) {
253
                    $extractorId = $attribute;
254
                } else {
255
                    $extractorId = $attributeParams['extractor'];
256
                }
257
258
                if (empty($attributeParams['value']) && $this->importer->hasExtractor($extractorId)) {
259
                    if (empty($relationsModels[$rowNbr]) || empty($relationsModels[$rowNbr][$attribute])) {
260
//                        $attributes[$attribute] = null;
261
                    } else {
262
                        $p = $relationsModels[$rowNbr][$attribute]->id;
263
                        $attributes[$attribute] = (int)$p;
264
                    }
265
                } else {
266
                    if (!empty($attributeParams['value'])) {
267
                        $attributes[$attribute] = $attributeParams['value'];
268
                    } else {
269
                        if (empty($attributeParams['column'])) {
270
                            throw new Exception('Not setted column for attribute ' . $attribute . ' for extractor ' . $this->id);
271
                        }
272
273
                        if (empty($row[$attributeParams['column'] - 1])) {
274
                            continue 2;
275
                        } else {
276
                            $value = $row[$attributeParams['column'] - 1];
277
                            if (!empty($attributeParams['numberDelimiter'])) {
278
                                if ($attributeParams['numberDelimiter'] == ',') {
279
                                    $value = str_replace('.', '', $value);
280
                                    $value = str_replace(',', '.', $value);
281
                                } else if ($attributeParams['numberDelimiter'] == '.') {
282
                                    $value = str_replace(',', '', $value);
283
                                }
284
                            }
285
286
                            $attributes[$attribute] = $value;
287
                        }
288
                    }
289
                }
290
            }
291
292
            $attributesNames = $this->getAttributesNamesForFind();
293
            $searchedAttributes = [];
294
            foreach ($attributesNames as $attributesName) {
295
                $modelClass = $this->query->modelClass;
296
                if (empty($attributes[$attributesName])) {
297
                    $value = null;
298
                } else {
299
                    $value = $attributes[$attributesName];
300
                }
301
302
                if (method_exists($modelClass, 'filtrateAttribute')) {
303
                    $value = $modelClass::filtrateAttribute($attributesName, $value);
304
                    $attributes[$attributesName] = $value;
305
                }
306
307
                $searchedAttributes[$attributesName] = $value;
308
            }
309
310
            $uniqueKey = serialize($searchedAttributes);
311
            if (isset($this->modelsByUniqueKey[$uniqueKey])) {
312
                $model = $this->modelsByUniqueKey[$uniqueKey];
313
            } else {
314
                $model = new $this->query->modelClass;
315
                $this->modelsByUniqueKey[$uniqueKey] = $model;
316
            }
317
318
            if ($this->isDelete && !$model->isNewRecord) {
319
                unset($this->deletedIds[$model->id]);
320
            }
321
322
            $modelAttributes = $model->getAttributes(array_keys($attributes));
323
            if (!$isSave || $model->isNewRecord || array_diff($attributes, $modelAttributes)) {
324
                $model->attributes = $attributes;
325
326
                $models[$rowNbr] = $model;
327
            }
328
        }
329
330
        $this->endOperation('models collect');
331
332
        $this->endOperation('extract');
333
        if ($isSave) {
334
            foreach ($models as $rowNbr => $model) {
335
                if (!$this->isBadRow($rowNbr)) {
336
                    $this->saveModel($model, $rowNbr);
337
                }
338
            }
339
        }
340
341
        return $models;
342
    }
343
344
    protected $times = [];
345
    protected function startOperation($name) {
346
        echo 'start ' . $name . ' ' . $this->id . "\n";
347
        $this->times[$name] = microtime(true);
348
    }
349
350
    protected function endOperation($name) {
351
        $time = microtime(true) - $this->times[$name];
352
        echo 'end ' . $name . ' ' . $this->id . ' after ' . $time . ' seconds' . "\n";
353
    }
354
355
    protected function triggerOperation($name) {
356
        echo $name . ' ' . $this->id . "\n";
357
    }
358
359
    protected function isBadRow($rowNbr) {
360
        return $this->importer->isBadRow($rowNbr);
361
    }
362
363
    protected function logError($message, $rowNbr, $model, $columnNbr = null, $isMarkBad = true) {
364
        if ($isMarkBad) {
365
            $this->importer->setIsBadRow($rowNbr);
366
        }
367
368
        $this->importer->logError($message, $rowNbr, $model, $columnNbr);
369
    }
370
371
    /**
372
     * @return array
373
     */
374
    public function getAttributesNamesForFind(): array
375
    {
376
        $attributesNames = [];
377
        foreach ($this->attributes as $attribute => $attributeParams) {
378
            if (!empty($attributeParams['isFind'])) {
379
                $attributesNames[] = $attribute;
380
            }
381
        }
382
        return $attributesNames;
383
    }
384
385
    /**
386
     * @param $model
387
     * @return mixed
388
     */
389
    protected function saveModel($model, $rowNbr)
390
    {
391
        $currentRowNbr = $this->importer->getCurrentStackRowNbr() + $rowNbr;
392
        $modelString = $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes));
393
        echo 'Row #' . $currentRowNbr . ': ';
394
        if (!$model->validate()) {
395
            $message = 'Error validate ' . $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)) . ' ' . serialize($model->errors);
396
            $this->logError($message, $rowNbr, $model);
397
            return false;
398
        }
399
400
        $isNewRecord = ($model->isNewRecord && !$this->isNoCreate);
401
        $isUpdatedRecord = (!$model->isNewRecord && !$this->isNoUpdate && !empty($model->dirtyAttributes));
402
403
        if ($isNewRecord || $isUpdatedRecord) {
404
            if ($isNewRecord) {
405
                $reason = 'created';
406
            } else {
407
                $reason = 'updated';
408
            }
409
410
            echo 'Saving ' . $modelString . ' because they is ' . $reason . "\n";
411
            if ($isUpdatedRecord) {
412
                echo 'Changed attributes ' . serialize(array_keys($model->dirtyAttributes)) . "\n";
413
                $oldValues = [];
414
                foreach ($model->dirtyAttributes as $attribute => $value) {
415
                    $oldValues[$attribute] = $model->oldAttributes[$attribute];
416
                }
417
418
                echo 'Old values ' . serialize($oldValues) . "\n";
419
                echo 'New values ' . serialize($model->dirtyAttributes) . "\n";
420
            }
421
422
            if (!$model->save()) {
423
                $this->logError('Error while saving ' . $modelString, $rowNbr, $model);
424
            }
425
        } else {
426
            echo $modelString . ' Is skipped because is not changed' . "\n";
427
        }
428
429
        return true;
430
    }
431
}