ModelsExtractor::getModels()   F
last analyzed

Complexity

Conditions 69
Paths > 20000

Size

Total Lines 276
Code Lines 172

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 0
CRAP Score 4830

Importance

Changes 2
Bugs 0 Features 0
Metric Value
eloc 172
c 2
b 0
f 0
dl 0
loc 276
ccs 0
cts 161
cp 0
rs 0
cc 69
nc 508202
nop 2
crap 4830

How to fix   Long Method    Complexity   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
<?php
2
/**
3
 * Created by PhpStorm.
4
 * User: execut
5
 * Date: 12/22/17
6
 * Time: 12:57 PM
7
 */
8
9
namespace execut\import\components;
10
11
12
use execut\import\ModelInterface;
13
use execut\import\Query;
14
use yii\base\Component;
15
use yii\base\Exception;
16
use yii\db\ActiveQuery;
17
use yii\db\ActiveRecord;
18
use yii\helpers\ArrayHelper;
19
20
class ModelsExtractor extends Component
21
{
22
    public $id = '';
23
24
    /**
25
     * @var ActiveQuery $query
26
     */
27
    public $query;
28
    public $scopes = null;
29
    public $isImport = false;
30
    public $attributes = [];
31
    /**
32
     * @var Importer
33
     */
34
    public $importer = null;
35
    protected $modelsByUniqueKey = [];
36
    public $isNoCreate = false;
37
    public $isNoUpdate = false;
38
    public $isDelete = false;
39
    public $deletedIds = [];
40
    public $uniqueKeys = null;
41
42
    public function reset() {
43
        $this->modelsByUniqueKey = [];
44
    }
45
46
    public function deleteOldRecords() {
47
        if ($this->isDelete) {
48
            $ids = $this->deletedIds;
49
            if (!empty($ids)) {
50
                if (count($ids) > 500000) {
51
                    throw new Exception('Many than 500 000 records to delete. Dangerous situation.');
52
                }
53
54
                $modelClass = $this->query->modelClass;
55
                while ($idsPart = array_splice($ids, 0, 65534)) {
56
                    if (count($idsPart) > 0) {
57
                        $modelClass::deleteAll([
58
                            'id' => $idsPart
59
                        ]);
60
                    }
61
                }
62
            }
63
        }
64
    }
65
66
    public function getModels($isSave = true, $isMarkBad = true) {
67
        /**
68
         * @var ActiveRecord $model
69
         */
70
        $this->startOperation('extract');
71
        $whereValues = [];
72
        $relationsModels = [];
73
        $this->startOperation('construct where');
74
        foreach ($this->attributes as $attribute => $attributeParams) {
75
            if (empty($attributeParams['extractor'])) {
76
                $extractorId = $attribute;
77
            } else {
78
                $extractorId = $attributeParams['extractor'];
79
            }
80
81
            if (empty($attributeParams['value']) && ($extractor = $this->importer->getExtractor($extractorId))) {
82
                $models = $extractor->getModels(false, $isMarkBad && !empty($attributeParams['isFind']));
83
                foreach ($this->importer->rows as $rowNbr => $row) {
0 ignored issues
show
Bug Best Practice introduced by eXeCUT
The property rows does not exist on execut\import\components\Importer. Since you implemented __get, consider adding a @property annotation.
Loading history...
84
                    if ($this->isBadRow($rowNbr)) {
85
                        continue;
86
                    }
87
88
                    if (empty($models[$rowNbr])) {
89
                        if ($isMarkBad && !empty($attributeParams['isFind'])) {
90
                            $this->importer->setIsBadRow($rowNbr);
91
                        }
92
93
                        continue;
94
                    }
95
96
                    $model = $models[$rowNbr];
97
98
                    if (empty($relationsModels[$rowNbr])) {
99
                        $relationsModels[$rowNbr] = [];
100
                    }
101
102
                    if (empty($extractor->isNoCreate) || !$model->isNewRecord) {
103
                        $relationsModels[$rowNbr][$attribute] = $model;
104
                    }
105
106
                    if (!empty($attributeParams['isFind'])) {
107
                        if (!$model->isNewRecord && !empty($model->dirtyAttributes)) {
108
                            if (empty($extractor->isNoUpdate)) {
109
                                if (!$this->saveModel($model, $rowNbr)) {
110
                                    unset($whereValues[$rowNbr]);
111
                                    continue;
112
                                }
113
                            }
114
                        } else if ($model->isNewRecord) {
115
                            if (empty($extractor->isNoCreate)) {
116
                                // Поиск не нужен, модель новая
117
                                unset($whereValues[$rowNbr]);
118
                                $this->saveModel($model, $rowNbr);
119
                                continue;
120
                            }
121
                        }
122
123
                        if ($model->isNewRecord && !empty($extractor->isNoCreate)) {
124
                            $this->logError('Related record is not founded with attributes ' . serialize(array_filter($model->attributes)), $rowNbr, $model, null, $isMarkBad);
125
                            unset($whereValues[$rowNbr]);
126
                            continue;
127
                        }
128
129
                        if (empty($whereValues[$rowNbr])) {
130
                            $whereValues[$rowNbr] = [];
131
                        }
132
133
                        $whereValues[$rowNbr][$attribute] = (int)$model->id;
134
                    }
135
                }
136
            } else {
137
                if (!empty($attributeParams['isFind'])) {
138
                    foreach ($this->importer->rows as $rowNbr => $row) {
139
                        if ($this->isBadRow($rowNbr)) {
140
                            continue;
141
                        }
142
143
                        if (!empty($attributeParams['value'])) {
144
                            $whereValues[$rowNbr][$attribute] = $attributeParams['value'];
145
                            continue;
146
                        }
147
148
                        if (empty($attributeParams['column'])) {
149
                            return [];
150
                            throw new Exception('Column key is required for attribute params. Extractor: ' . $extractorId . '. Attribute params: ' . var_export($attributeParams, true));
0 ignored issues
show
Unused Code introduced by eXeCUT
ThrowNode is not reachable.

This check looks for unreachable code. It uses sophisticated control flow analysis techniques to find statements which will never be executed.

Unreachable code is most often the result of return, die or exit statements that have been added for debug purposes.

function fx() {
    try {
        doSomething();
        return true;
    }
    catch (\Exception $e) {
        return false;
    }

    return false;
}

In the above example, the last return false will never be executed, because a return statement has already been met in every possible execution path.

Loading history...
151
                        }
152
153
                        if (empty($row[$attributeParams['column'] - 1])) {
154
                            $this->logError($attribute . ' is required for record find', $rowNbr, null, $attributeParams['column']);
155
                            unset($whereValues[$rowNbr]);
156
                            continue;
157
                        }
158
159
                        if (empty($whereValues[$rowNbr])) {
160
                            $whereValues[$rowNbr] = [];
161
                        }
162
163
                        $modelClass = $this->query->modelClass;
164
                        $value = $row[$attributeParams['column'] - 1];
165
                        if (method_exists($modelClass, 'filtrateAttribute')) {
166
                            $value = $modelClass::filtrateAttribute($attribute, $value);
167
                        }
168
169
                        $whereValues[$rowNbr][$attribute] = $value;
170
                    }
171
                }
172
            }
173
        }
174
175
        $result = [];
176
        foreach ($whereValues as $rowNbr => $whereValue) {
177
            if ($this->importer->isBadRow($rowNbr)) {
178
                continue;
179
            }
180
181
            $uniqueKey = serialize($whereValue);
182
            if (!isset($this->modelsByUniqueKey[$uniqueKey])) {
183
                $result[$uniqueKey] = $whereValue;
184
            }
185
        }
186
187
        $whereValues = $result;
188
189
        $this->endOperation('construct where');
190
        if (count($whereValues)) {
191
            $attributesNames = $this->getAttributesNamesForFind();
192
            $query = clone $this->query;
193
            $query->indexBy(function ($row) use ($attributesNames) {
194
                $searchedAttributes = [];
195
                foreach ($attributesNames as $attributesName) {
196
                    $modelClass = $this->query->modelClass;
197
                    $value = $row[$attributesName];
198
                    if (method_exists($modelClass, 'filtrateAttribute')) {
199
                        $value = $modelClass::filtrateAttribute($attributesName, $value);
200
                    }
201
202
                    $searchedAttributes[$attributesName] = $value;
203
                }
204
                return serialize($searchedAttributes);
205
            });
206
207
            if ($this->scopes !== null) {
208
                foreach ($this->scopes as $scope) {
209
                    $scope($query, [
210
                        'IN',
211
                        $attributesNames,
212
                        $whereValues,
213
                    ]);
214
                }
215
            } else {
216
                $query->andWhere([
217
                    'IN',
218
                    $attributesNames,
219
                    $whereValues,
220
                ]);
221
            }
222
223
            $this->startOperation('find');
224
            $models = $query->all();
225
            $this->endOperation('find');
226
            $this->startOperation('keys collect');
227
            foreach ($models as $uniqueKey => $model) {
228
                if ($this->uniqueKeys !== null) {
229
                    $callback = $this->uniqueKeys;
230
                    $uniqueKeys = $callback($model, $attributesNames, $whereValues);
231
                } else {
232
                    $uniqueKeys = [$uniqueKey];
233
                }
234
235
                foreach ($uniqueKeys as $uniqueKey) {
0 ignored issues
show
Comprehensibility Bug introduced by eXeCUT
$uniqueKey is overwriting a variable from outer foreach loop.
Loading history...
236
                    $this->modelsByUniqueKey[$uniqueKey] = $model;
237
                }
238
            }
239
240
            $this->endOperation('keys collect');
241
        }
242
243
        $models = [];
244
        $this->startOperation('models collect');
245
        foreach ($this->importer->rows as $rowNbr => $row) {
246
            if ($this->isBadRow($rowNbr)) {
247
                continue;
248
            }
249
250
            $attributes = [];
251
            foreach ($this->attributes as $attribute => $attributeParams) {
252
                if (empty($attributeParams['extractor'])) {
253
                    $extractorId = $attribute;
254
                } else {
255
                    $extractorId = $attributeParams['extractor'];
256
                }
257
258
                if (empty($attributeParams['value']) && $this->importer->hasExtractor($extractorId)) {
259
                    if (empty($relationsModels[$rowNbr]) || empty($relationsModels[$rowNbr][$attribute])) {
260
//                        $attributes[$attribute] = null;
261
                    } else {
262
                        $p = $relationsModels[$rowNbr][$attribute]->id;
263
                        $attributes[$attribute] = (int)$p;
264
                    }
265
                } else {
266
                    if (!empty($attributeParams['value'])) {
267
                        $attributes[$attribute] = $attributeParams['value'];
268
                    } else {
269
                        if (empty($attributeParams['column'])) {
270
                            throw new Exception('Not setted column for attribute ' . $attribute . ' for extractor ' . $this->id);
271
                        }
272
273
                        if (empty($row[$attributeParams['column'] - 1])) {
274
                            continue 2;
275
                        } else {
276
                            $value = $row[$attributeParams['column'] - 1];
277
                            if (!empty($attributeParams['numberDelimiter'])) {
278
                                if ($attributeParams['numberDelimiter'] == ',') {
279
                                    $value = str_replace('.', '', $value);
280
                                    $value = str_replace(',', '.', $value);
281
                                } else if ($attributeParams['numberDelimiter'] == '.') {
282
                                    $value = str_replace(',', '', $value);
283
                                }
284
                            }
285
286
                            $attributes[$attribute] = $value;
287
                        }
288
                    }
289
                }
290
            }
291
292
            $attributesNames = $this->getAttributesNamesForFind();
293
            $searchedAttributes = [];
294
            foreach ($attributesNames as $attributesName) {
295
                $modelClass = $this->query->modelClass;
296
                if (empty($attributes[$attributesName])) {
297
                    $value = null;
298
                } else {
299
                    $value = $attributes[$attributesName];
300
                }
301
302
                if (method_exists($modelClass, 'filtrateAttribute')) {
303
                    $value = $modelClass::filtrateAttribute($attributesName, $value);
304
                    $attributes[$attributesName] = $value;
305
                }
306
307
                $searchedAttributes[$attributesName] = $value;
308
            }
309
310
            $uniqueKey = serialize($searchedAttributes);
311
            if (isset($this->modelsByUniqueKey[$uniqueKey])) {
312
                $model = $this->modelsByUniqueKey[$uniqueKey];
313
            } else {
314
                $model = new $this->query->modelClass;
315
                $this->modelsByUniqueKey[$uniqueKey] = $model;
316
            }
317
318
            if ($this->isDelete && !$model->isNewRecord) {
319
                unset($this->deletedIds[$model->id]);
320
            }
321
322
            $modelAttributes = $model->getAttributes(array_keys($attributes));
323
            if (!$isSave || $model->isNewRecord || array_diff($attributes, $modelAttributes)) {
324
                $model->attributes = $attributes;
325
326
                $models[$rowNbr] = $model;
327
            }
328
        }
329
330
        $this->endOperation('models collect');
331
332
        $this->endOperation('extract');
333
        if ($isSave) {
334
            foreach ($models as $rowNbr => $model) {
335
                if (!$this->isBadRow($rowNbr)) {
336
                    $this->saveModel($model, $rowNbr);
337
                }
338
            }
339
        }
340
341
        return $models;
342
    }
343
344
    protected $times = [];
345
    protected function startOperation($name) {
346
        echo 'start ' . $name . ' ' . $this->id . "\n";
347
        $this->times[$name] = microtime(true);
348
    }
349
350
    protected function endOperation($name) {
351
        $time = microtime(true) - $this->times[$name];
352
        echo 'end ' . $name . ' ' . $this->id . ' after ' . $time . ' seconds' . "\n";
353
    }
354
355
    protected function triggerOperation($name) {
356
        echo $name . ' ' . $this->id . "\n";
357
    }
358
359
    protected function isBadRow($rowNbr) {
360
        return $this->importer->isBadRow($rowNbr);
361
    }
362
363
    protected function logError($message, $rowNbr, $model, $columnNbr = null, $isMarkBad = true) {
364
        if ($isMarkBad) {
365
            $this->importer->setIsBadRow($rowNbr);
366
        }
367
368
        $this->importer->logError($message, $rowNbr, $model, $columnNbr);
369
    }
370
371
    /**
372
     * @return array
373
     */
374
    public function getAttributesNamesForFind(): array
375
    {
376
        $attributesNames = [];
377
        foreach ($this->attributes as $attribute => $attributeParams) {
378
            if (!empty($attributeParams['isFind'])) {
379
                $attributesNames[] = $attribute;
380
            }
381
        }
382
        return $attributesNames;
383
    }
384
385
    /**
386
     * @param $model
387
     * @return mixed
388
     */
389
    protected function saveModel($model, $rowNbr)
390
    {
391
        $currentRowNbr = $this->importer->getCurrentStackRowNbr() + $rowNbr;
392
        $modelString = $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes));
393
        echo 'Row #' . $currentRowNbr . ': ';
394
        if (!$model->validate()) {
395
            $message = 'Error validate ' . $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)) . ' ' . serialize($model->errors);
396
            $this->logError($message, $rowNbr, $model);
397
            return false;
398
        }
399
400
        $isNewRecord = ($model->isNewRecord && !$this->isNoCreate);
401
        $isUpdatedRecord = (!$model->isNewRecord && !$this->isNoUpdate && !empty($model->dirtyAttributes));
402
403
        if ($isNewRecord || $isUpdatedRecord) {
404
            if ($isNewRecord) {
405
                $reason = 'created';
406
            } else {
407
                $reason = 'updated';
408
            }
409
410
            echo 'Saving ' . $modelString . ' because they is ' . $reason . "\n";
411
            if ($isUpdatedRecord) {
412
                echo 'Changed attributes ' . serialize(array_keys($model->dirtyAttributes)) . "\n";
413
                $oldValues = [];
414
                foreach ($model->dirtyAttributes as $attribute => $value) {
415
                    $oldValues[$attribute] = $model->oldAttributes[$attribute];
416
                }
417
418
                echo 'Old values ' . serialize($oldValues) . "\n";
419
                echo 'New values ' . serialize($model->dirtyAttributes) . "\n";
420
            }
421
422
            if (!$model->save()) {
423
                $this->logError('Error while saving ' . $modelString, $rowNbr, $model);
424
            }
425
        } else {
426
            echo $modelString . ' Is skipped because is not changed' . "\n";
427
        }
428
429
        return true;
430
    }
431
}