Passed
Push — master ( 662d52...2a8232 )
by eXeCUT
03:58
created

components/ModelsExtractor.php (3 issues)

1
<?php
2
/**
3
 * Created by PhpStorm.
4
 * User: execut
5
 * Date: 12/22/17
6
 * Time: 12:57 PM
7
 */
8
9
namespace execut\import\components;
10
11
12
use execut\import\ModelInterface;
13
use execut\import\Query;
14
use yii\base\Component;
15
use yii\base\Exception;
16
use yii\db\ActiveQuery;
17
use yii\db\ActiveRecord;
18
use yii\helpers\ArrayHelper;
19
20
class ModelsExtractor extends Component
21
{
22
    public $id = '';
23
24
    /**
25
     * @var ActiveQuery $query
26
     */
27
    public $query;
28
    public $scopes = null;
29
    public $isImport = false;
30
    public $attributes = [];
31
    /**
32
     * @var Importer
33
     */
34
    public $importer = null;
35
    protected $modelsByUniqueKey = [];
36
    public $isNoCreate = false;
37
    public $isNoUpdate = false;
38
    public $isDelete = false;
39
    public $deletedIds = [];
40
    public $uniqueKeys = null;
41
42
    public function reset() {
43
        $this->modelsByUniqueKey = [];
44
    }
45
46
    public function deleteOldRecords() {
47
        if ($this->isDelete) {
48
            $ids = $this->deletedIds;
49
            if (!empty($ids)) {
50
                if (count($ids) > 500000) {
51
                    throw new Exception('Many than 500 000 records to delete. Dangerous situation.');
52
                }
53
54
                $modelClass = $this->query->modelClass;
55
                while ($idsPart = array_splice($ids, 0, 65534)) {
56
                    if (count($idsPart) > 0) {
57
                        $modelClass::deleteAll([
58
                            'id' => $idsPart
59
                        ]);
60
                    }
61
                }
62
            }
63
        }
64
    }
65
66
    public function getModels($isSave = true, $isMarkBad = true) {
67
        /**
68
         * @var ActiveRecord $model
69
         */
70
        $this->startOperation('extract');
71
        $whereValues = [];
72
        $relationsModels = [];
73
        $this->startOperation('construct where');
74
        foreach ($this->attributes as $attribute => $attributeParams) {
75
            if (empty($attributeParams['extractor'])) {
76
                $extractorId = $attribute;
77
            } else {
78
                $extractorId = $attributeParams['extractor'];
79
            }
80
81
            if (empty($attributeParams['value']) && ($extractor = $this->importer->getExtractor($extractorId))) {
82
                $models = $extractor->getModels(false, $isMarkBad && !empty($attributeParams['isFind']));
83
                foreach ($this->importer->rows as $rowNbr => $row) {
0 ignored issues
show
Bug Best Practice introduced by eXeCUT
The property rows does not exist on execut\import\components\Importer. Since you implemented __get, consider adding a @property annotation.
Loading history...
84
                    if ($this->isBadRow($rowNbr)) {
85
                        continue;
86
                    }
87
88
                    if (empty($models[$rowNbr])) {
89
                        if ($isMarkBad && !empty($attributeParams['isFind'])) {
90
                            $this->importer->setIsBadRow($rowNbr);
91
                        }
92
93
                        continue;
94
                    }
95
96
                    $model = $models[$rowNbr];
97
98
                    if (empty($relationsModels[$rowNbr])) {
99
                        $relationsModels[$rowNbr] = [];
100
                    }
101
102
                    if (empty($extractor->isNoCreate) || !$model->isNewRecord) {
103
                        $relationsModels[$rowNbr][$attribute] = $model;
104
                    }
105
106
                    if (!empty($attributeParams['isFind'])) {
107
                        if (!$model->isNewRecord && !empty($model->dirtyAttributes)) {
108
                            if (empty($extractor->isNoUpdate)) {
109
                                if (!$this->saveModel($model, $rowNbr)) {
110
                                    unset($whereValues[$rowNbr]);
111
                                    continue;
112
                                }
113
                            }
114
                        } else if ($model->isNewRecord) {
115
                            if (empty($extractor->isNoCreate)) {
116
                                // Поиск не нужен, модель новая
117
                                unset($whereValues[$rowNbr]);
118
                                $this->saveModel($model, $rowNbr);
119
                                continue;
120
                            }
121
                        }
122
123
                        if ($model->isNewRecord && !empty($extractor->isNoCreate)) {
124
                            $this->logError('Related record is not founded with attributes ' . serialize(array_filter($model->attributes)), $rowNbr, $model, null, $isMarkBad);
125
                            unset($whereValues[$rowNbr]);
126
                            continue;
127
                        }
128
129
                        if (empty($whereValues[$rowNbr])) {
130
                            $whereValues[$rowNbr] = [];
131
                        }
132
133
                        $whereValues[$rowNbr][$attribute] = (int)$model->id;
134
                    }
135
                }
136
            } else {
137
                if (!empty($attributeParams['isFind'])) {
138
                    foreach ($this->importer->rows as $rowNbr => $row) {
139
                        if ($this->isBadRow($rowNbr)) {
140
                            continue;
141
                        }
142
143
                        if (!empty($attributeParams['value'])) {
144
                            $whereValues[$rowNbr][$attribute] = $attributeParams['value'];
145
                            continue;
146
                        }
147
148
                        if (empty($attributeParams['column'])) {
149
                            return [];
150
                            throw new Exception('Column key is required for attribute params. Extractor: ' . $extractorId . '. Attribute params: ' . var_export($attributeParams, true));
0 ignored issues
show
ThrowNode is not reachable.

This check looks for unreachable code. It uses sophisticated control flow analysis techniques to find statements which will never be executed.

Unreachable code is most often the result of return, die or exit statements that have been added for debug purposes.

function fx() {
    try {
        doSomething();
        return true;
    }
    catch (\Exception $e) {
        return false;
    }

    return false;
}

In the above example, the last return false will never be executed, because a return statement has already been met in every possible execution path.

Loading history...
151
                        }
152
153
                        if (empty($row[$attributeParams['column'] - 1])) {
154
                            $this->logError($attribute . ' is required for record find', $rowNbr, null, $attributeParams['column']);
155
                            unset($whereValues[$rowNbr]);
156
                            continue;
157
                        }
158
159
                        if (empty($whereValues[$rowNbr])) {
160
                            $whereValues[$rowNbr] = [];
161
                        }
162
163
                        $modelClass = $this->query->modelClass;
164
                        $value = $row[$attributeParams['column'] - 1];
165
                        if (method_exists($modelClass, 'filtrateAttribute')) {
166
                            $value = $modelClass::filtrateAttribute($attribute, $value);
167
                        }
168
169
                        $whereValues[$rowNbr][$attribute] = $value;
170
                    }
171
                }
172
            }
173
        }
174
175
        $result = [];
176
        foreach ($whereValues as $rowNbr => $whereValue) {
177
            if ($this->importer->isBadRow($rowNbr)) {
178
                continue;
179
            }
180
181
            $uniqueKey = serialize($whereValue);
182
            if (!isset($this->modelsByUniqueKey[$uniqueKey])) {
183
                $result[$uniqueKey] = $whereValue;
184
            }
185
        }
186
187
        $whereValues = $result;
188
189
        $this->endOperation('construct where');
190
        if (count($whereValues)) {
191
            $attributesNames = $this->getAttributesNamesForFind();
192
            $query = clone $this->query;
193
            $query->indexBy(function ($row) use ($attributesNames) {
194
                $searchedAttributes = [];
195
                foreach ($attributesNames as $attributesName) {
196
                    $modelClass = $this->query->modelClass;
197
                    $value = $row[$attributesName];
198
                    if (method_exists($modelClass, 'filtrateAttribute')) {
199
                        $value = $modelClass::filtrateAttribute($attributesName, $value);
200
                    }
201
202
                    $searchedAttributes[$attributesName] = $value;
203
                }
204
                return serialize($searchedAttributes);
205
            });
206
207
            if ($this->scopes !== null) {
208
                foreach ($this->scopes as $scope) {
209
                    $scope($query, [
210
                        'IN',
211
                        $attributesNames,
212
                        $whereValues,
213
                    ]);
214
                }
215
            } else {
216
                $query->andWhere([
217
                    'IN',
218
                    $attributesNames,
219
                    $whereValues,
220
                ]);
221
            }
222
223
            $this->startOperation('find');
224
            $models = $query->all();
225
            $this->endOperation('find');
226
            $this->startOperation('keys collect');
227
            foreach ($models as $uniqueKey => $model) {
228
                if ($this->uniqueKeys !== null) {
229
                    $callback = $this->uniqueKeys;
230
                    $uniqueKeys = $callback($model, $attributesNames, $whereValues);
231
                } else {
232
                    $uniqueKeys = [$uniqueKey];
233
                }
234
235
                foreach ($uniqueKeys as $uniqueKey) {
0 ignored issues
show
Comprehensibility Bug introduced by eXeCUT
$uniqueKey is overwriting a variable from outer foreach loop.
Loading history...
236
                    $this->modelsByUniqueKey[$uniqueKey] = $model;
237
                }
238
            }
239
240
            $this->endOperation('keys collect');
241
        }
242
243
        $models = [];
244
        $this->startOperation('models collect');
245
        foreach ($this->importer->rows as $rowNbr => $row) {
246
            if ($this->isBadRow($rowNbr)) {
247
                continue;
248
            }
249
250
            $attributes = [];
251
            foreach ($this->attributes as $attribute => $attributeParams) {
252
                if (empty($attributeParams['extractor'])) {
253
                    $extractorId = $attribute;
254
                } else {
255
                    $extractorId = $attributeParams['extractor'];
256
                }
257
258
                if (empty($attributeParams['value']) && $this->importer->hasExtractor($extractorId)) {
259
                    if (empty($relationsModels[$rowNbr]) || empty($relationsModels[$rowNbr][$attribute])) {
260
//                        $attributes[$attribute] = null;
261
                    } else {
262
                        $p = $relationsModels[$rowNbr][$attribute]->id;
263
                        $attributes[$attribute] = (int)$p;
264
                    }
265
                } else {
266
                    if (!empty($attributeParams['value'])) {
267
                        $attributes[$attribute] = $attributeParams['value'];
268
                    } else {
269
                        if (empty($attributeParams['column'])) {
270
                            throw new Exception('Not setted column for attribute ' . $attribute . ' for extractor ' . $this->id);
271
                        }
272
273
                        if (empty($row[$attributeParams['column'] - 1])) {
274
                            continue 2;
275
                        } else {
276
                            $attributes[$attribute] = $row[$attributeParams['column'] - 1];
277
                        }
278
                    }
279
                }
280
            }
281
282
            $attributesNames = $this->getAttributesNamesForFind();
283
            $searchedAttributes = [];
284
            foreach ($attributesNames as $attributesName) {
285
                $modelClass = $this->query->modelClass;
286
                if (empty($attributes[$attributesName])) {
287
                    $value = null;
288
                } else {
289
                    $value = $attributes[$attributesName];
290
                }
291
292
                if (method_exists($modelClass, 'filtrateAttribute')) {
293
                    $value = $modelClass::filtrateAttribute($attributesName, $value);
294
                    $attributes[$attributesName] = $value;
295
                }
296
297
                $searchedAttributes[$attributesName] = $value;
298
            }
299
300
            $uniqueKey = serialize($searchedAttributes);
301
            if (isset($this->modelsByUniqueKey[$uniqueKey])) {
302
                $model = $this->modelsByUniqueKey[$uniqueKey];
303
            } else {
304
                $model = new $this->query->modelClass;
305
                $this->modelsByUniqueKey[$uniqueKey] = $model;
306
            }
307
308
            if ($this->isDelete && !$model->isNewRecord) {
309
                unset($this->deletedIds[$model->id]);
310
            }
311
312
            $modelAttributes = $model->getAttributes(array_keys($attributes));
313
            if (!$isSave || $model->isNewRecord || array_diff($attributes, $modelAttributes)) {
314
                $model->attributes = $attributes;
315
316
                $models[$rowNbr] = $model;
317
            }
318
319
//            if ($model->isNewRecord && empty($this->isNoCreate)) {
320
//                echo 1;
321
//            }
322
        }
323
324
        $this->endOperation('models collect');
325
326
        $this->endOperation('extract');
327
        if ($isSave) {
328
            foreach ($models as $rowNbr => $model) {
329
                if (!$this->isBadRow($rowNbr)) {
330
                    $this->saveModel($model, $rowNbr);
331
                }
332
            }
333
        }
334
335
        return $models;
336
    }
337
338
    protected $times = [];
339
    protected function startOperation($name) {
340
        echo 'start ' . $name . ' ' . $this->id . "\n";
341
        $this->times[$name] = microtime(true);
342
    }
343
344
    protected function endOperation($name) {
345
        $time = microtime(true) - $this->times[$name];
346
        echo 'end ' . $name . ' ' . $this->id . ' after ' . $time . ' seconds' . "\n";
347
    }
348
349
    protected function triggerOperation($name) {
350
        echo $name . ' ' . $this->id . "\n";
351
    }
352
353
    protected function isBadRow($rowNbr) {
354
        return $this->importer->isBadRow($rowNbr);
355
    }
356
357
    protected function logError($message, $rowNbr, $model, $columnNbr = null, $isMarkBad = true) {
358
        if ($isMarkBad) {
359
            $this->importer->setIsBadRow($rowNbr);
360
        }
361
362
        $this->importer->logError($message, $rowNbr, $model, $columnNbr);
363
    }
364
365
    /**
366
     * @return array
367
     */
368
    public function getAttributesNamesForFind(): array
369
    {
370
        $attributesNames = [];
371
        foreach ($this->attributes as $attribute => $attributeParams) {
372
            if (!empty($attributeParams['isFind'])) {
373
                $attributesNames[] = $attribute;
374
            }
375
        }
376
        return $attributesNames;
377
    }
378
379
    /**
380
     * @param $model
381
     * @return mixed
382
     */
383
    protected function saveModel($model, $rowNbr)
384
    {
385
        $currentRowNbr = $this->importer->getCurrentStackRowNbr() + $rowNbr;
386
        $modelString = $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes));
387
        echo 'Row #' . $currentRowNbr . ': ';
388
        if (!$model->validate()) {
389
            $message = 'Error validate ' . $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)) . ' ' . serialize($model->errors);
390
            $this->logError($message, $rowNbr, $model);
391
            return false;
392
        }
393
394
        $isNewRecord = ($model->isNewRecord && !$this->isNoCreate);
395
        $isUpdatedRecord = (!$model->isNewRecord && !$this->isNoUpdate && !empty($model->dirtyAttributes));
396
        if ($isNewRecord || $isUpdatedRecord) {
397
            if ($isNewRecord) {
398
                $reason = 'created';
399
            } else {
400
                $reason = 'updated';
401
            }
402
403
            echo 'Saving ' . $modelString . ' because they is ' . $reason . "\n";
404
            if ($isUpdatedRecord) {
405
                echo 'Changed attributes ' . serialize(array_keys($model->dirtyAttributes)) . "\n";
406
                $oldValues = [];
407
                foreach ($model->dirtyAttributes as $attribute => $value) {
408
                    $oldValues[$attribute] = $model->oldAttributes[$attribute];
409
                }
410
411
                echo 'Old values ' . serialize($oldValues) . "\n";
412
                echo 'New values ' . serialize($model->dirtyAttributes) . "\n";
413
            }
414
415
            if (!$model->save()) {
416
                $this->logError('Error while saving ' . $modelString, $rowNbr, $model);
417
            }
418
        } else {
419
            echo $modelString . ' Is skipped because is not changed' . "\n";
420
        }
421
422
        return true;
423
    }
424
}