1 | <?php |
||
2 | /** |
||
3 | * Created by PhpStorm. |
||
4 | * User: execut |
||
5 | * Date: 12/22/17 |
||
6 | * Time: 12:57 PM |
||
7 | */ |
||
8 | |||
9 | namespace execut\import\components; |
||
10 | |||
11 | |||
12 | use execut\import\ModelInterface; |
||
13 | use execut\import\Query; |
||
14 | use yii\base\Component; |
||
15 | use yii\base\Exception; |
||
16 | use yii\db\ActiveQuery; |
||
17 | use yii\db\ActiveRecord; |
||
18 | use yii\helpers\ArrayHelper; |
||
19 | |||
20 | class ModelsExtractor extends Component |
||
21 | { |
||
22 | public $id = ''; |
||
23 | |||
24 | /** |
||
25 | * @var ActiveQuery $query |
||
26 | */ |
||
27 | public $query; |
||
28 | public $scopes = null; |
||
29 | public $isImport = false; |
||
30 | public $attributes = []; |
||
31 | /** |
||
32 | * @var Importer |
||
33 | */ |
||
34 | public $importer = null; |
||
35 | protected $modelsByUniqueKey = []; |
||
36 | public $isNoCreate = false; |
||
37 | public $isNoUpdate = false; |
||
38 | public $isDelete = false; |
||
39 | public $deletedIds = []; |
||
40 | public $uniqueKeys = null; |
||
41 | |||
42 | public function reset() { |
||
43 | $this->modelsByUniqueKey = []; |
||
44 | } |
||
45 | |||
46 | public function deleteOldRecords() { |
||
47 | if ($this->isDelete) { |
||
48 | $ids = $this->deletedIds; |
||
49 | if (!empty($ids)) { |
||
50 | if (count($ids) > 500000) { |
||
51 | throw new Exception('Many than 500 000 records to delete. Dangerous situation.'); |
||
52 | } |
||
53 | |||
54 | $modelClass = $this->query->modelClass; |
||
55 | while ($idsPart = array_splice($ids, 0, 65534)) { |
||
56 | if (count($idsPart) > 0) { |
||
57 | $modelClass::deleteAll([ |
||
58 | 'id' => $idsPart |
||
59 | ]); |
||
60 | } |
||
61 | } |
||
62 | } |
||
63 | } |
||
64 | } |
||
65 | |||
66 | public function getModels($isSave = true, $isMarkBad = true) { |
||
67 | /** |
||
68 | * @var ActiveRecord $model |
||
69 | */ |
||
70 | $this->startOperation('extract'); |
||
71 | $whereValues = []; |
||
72 | $relationsModels = []; |
||
73 | $this->startOperation('construct where'); |
||
74 | foreach ($this->attributes as $attribute => $attributeParams) { |
||
75 | if (empty($attributeParams['extractor'])) { |
||
76 | $extractorId = $attribute; |
||
77 | } else { |
||
78 | $extractorId = $attributeParams['extractor']; |
||
79 | } |
||
80 | |||
81 | if (empty($attributeParams['value']) && ($extractor = $this->importer->getExtractor($extractorId))) { |
||
82 | $models = $extractor->getModels(false, $isMarkBad && !empty($attributeParams['isFind'])); |
||
83 | foreach ($this->importer->rows as $rowNbr => $row) { |
||
0 ignored issues
–
show
Bug
Best Practice
introduced
by
Loading history...
|
|||
84 | if ($this->isBadRow($rowNbr)) { |
||
85 | continue; |
||
86 | } |
||
87 | |||
88 | if (empty($models[$rowNbr])) { |
||
89 | if ($isMarkBad && !empty($attributeParams['isFind'])) { |
||
90 | $this->importer->setIsBadRow($rowNbr); |
||
91 | } |
||
92 | |||
93 | continue; |
||
94 | } |
||
95 | |||
96 | $model = $models[$rowNbr]; |
||
97 | |||
98 | if (empty($relationsModels[$rowNbr])) { |
||
99 | $relationsModels[$rowNbr] = []; |
||
100 | } |
||
101 | |||
102 | if (empty($extractor->isNoCreate) || !$model->isNewRecord) { |
||
103 | $relationsModels[$rowNbr][$attribute] = $model; |
||
104 | } |
||
105 | |||
106 | if (!empty($attributeParams['isFind'])) { |
||
107 | if (!$model->isNewRecord && !empty($model->dirtyAttributes)) { |
||
108 | if (empty($extractor->isNoUpdate)) { |
||
109 | if (!$this->saveModel($model, $rowNbr)) { |
||
110 | unset($whereValues[$rowNbr]); |
||
111 | continue; |
||
112 | } |
||
113 | } |
||
114 | } else if ($model->isNewRecord) { |
||
115 | if (empty($extractor->isNoCreate)) { |
||
116 | // Поиск не нужен, модель новая |
||
117 | unset($whereValues[$rowNbr]); |
||
118 | $this->saveModel($model, $rowNbr); |
||
119 | continue; |
||
120 | } |
||
121 | } |
||
122 | |||
123 | if ($model->isNewRecord && !empty($extractor->isNoCreate)) { |
||
124 | $this->logError('Related record is not founded with attributes ' . serialize(array_filter($model->attributes)), $rowNbr, $model, null, $isMarkBad); |
||
125 | unset($whereValues[$rowNbr]); |
||
126 | continue; |
||
127 | } |
||
128 | |||
129 | if (empty($whereValues[$rowNbr])) { |
||
130 | $whereValues[$rowNbr] = []; |
||
131 | } |
||
132 | |||
133 | $whereValues[$rowNbr][$attribute] = (int)$model->id; |
||
134 | } |
||
135 | } |
||
136 | } else { |
||
137 | if (!empty($attributeParams['isFind'])) { |
||
138 | foreach ($this->importer->rows as $rowNbr => $row) { |
||
139 | if ($this->isBadRow($rowNbr)) { |
||
140 | continue; |
||
141 | } |
||
142 | |||
143 | if (!empty($attributeParams['value'])) { |
||
144 | $whereValues[$rowNbr][$attribute] = $attributeParams['value']; |
||
145 | continue; |
||
146 | } |
||
147 | |||
148 | if (empty($attributeParams['column'])) { |
||
149 | return []; |
||
150 | throw new Exception('Column key is required for attribute params. Extractor: ' . $extractorId . '. Attribute params: ' . var_export($attributeParams, true)); |
||
0 ignored issues
–
show
ThrowNode is not reachable.
This check looks for unreachable code. It uses sophisticated control flow analysis techniques to find statements which will never be executed. Unreachable code is most often the result of function fx() {
try {
doSomething();
return true;
}
catch (\Exception $e) {
return false;
}
return false;
}
In the above example, the last
Loading history...
|
|||
151 | } |
||
152 | |||
153 | if (empty($row[$attributeParams['column'] - 1])) { |
||
154 | $this->logError($attribute . ' is required for record find', $rowNbr, null, $attributeParams['column']); |
||
155 | unset($whereValues[$rowNbr]); |
||
156 | continue; |
||
157 | } |
||
158 | |||
159 | if (empty($whereValues[$rowNbr])) { |
||
160 | $whereValues[$rowNbr] = []; |
||
161 | } |
||
162 | |||
163 | $modelClass = $this->query->modelClass; |
||
164 | $value = $row[$attributeParams['column'] - 1]; |
||
165 | if (method_exists($modelClass, 'filtrateAttribute')) { |
||
166 | $value = $modelClass::filtrateAttribute($attribute, $value); |
||
167 | } |
||
168 | |||
169 | $whereValues[$rowNbr][$attribute] = $value; |
||
170 | } |
||
171 | } |
||
172 | } |
||
173 | } |
||
174 | |||
175 | $result = []; |
||
176 | foreach ($whereValues as $rowNbr => $whereValue) { |
||
177 | if ($this->importer->isBadRow($rowNbr)) { |
||
178 | continue; |
||
179 | } |
||
180 | |||
181 | $uniqueKey = serialize($whereValue); |
||
182 | if (!isset($this->modelsByUniqueKey[$uniqueKey])) { |
||
183 | $result[$uniqueKey] = $whereValue; |
||
184 | } |
||
185 | } |
||
186 | |||
187 | $whereValues = $result; |
||
188 | |||
189 | $this->endOperation('construct where'); |
||
190 | if (count($whereValues)) { |
||
191 | $attributesNames = $this->getAttributesNamesForFind(); |
||
192 | $query = clone $this->query; |
||
193 | $query->indexBy(function ($row) use ($attributesNames) { |
||
194 | $searchedAttributes = []; |
||
195 | foreach ($attributesNames as $attributesName) { |
||
196 | $modelClass = $this->query->modelClass; |
||
197 | $value = $row[$attributesName]; |
||
198 | if (method_exists($modelClass, 'filtrateAttribute')) { |
||
199 | $value = $modelClass::filtrateAttribute($attributesName, $value); |
||
200 | } |
||
201 | |||
202 | $searchedAttributes[$attributesName] = $value; |
||
203 | } |
||
204 | return serialize($searchedAttributes); |
||
205 | }); |
||
206 | |||
207 | if ($this->scopes !== null) { |
||
208 | foreach ($this->scopes as $scope) { |
||
209 | $scope($query, [ |
||
210 | 'IN', |
||
211 | $attributesNames, |
||
212 | $whereValues, |
||
213 | ]); |
||
214 | } |
||
215 | } else { |
||
216 | $query->andWhere([ |
||
217 | 'IN', |
||
218 | $attributesNames, |
||
219 | $whereValues, |
||
220 | ]); |
||
221 | } |
||
222 | |||
223 | $this->startOperation('find'); |
||
224 | $models = $query->all(); |
||
225 | $this->endOperation('find'); |
||
226 | $this->startOperation('keys collect'); |
||
227 | foreach ($models as $uniqueKey => $model) { |
||
228 | if ($this->uniqueKeys !== null) { |
||
229 | $callback = $this->uniqueKeys; |
||
230 | $uniqueKeys = $callback($model, $attributesNames, $whereValues); |
||
231 | } else { |
||
232 | $uniqueKeys = [$uniqueKey]; |
||
233 | } |
||
234 | |||
235 | foreach ($uniqueKeys as $uniqueKey) { |
||
0 ignored issues
–
show
|
|||
236 | $this->modelsByUniqueKey[$uniqueKey] = $model; |
||
237 | } |
||
238 | } |
||
239 | |||
240 | $this->endOperation('keys collect'); |
||
241 | } |
||
242 | |||
243 | $models = []; |
||
244 | $this->startOperation('models collect'); |
||
245 | foreach ($this->importer->rows as $rowNbr => $row) { |
||
246 | if ($this->isBadRow($rowNbr)) { |
||
247 | continue; |
||
248 | } |
||
249 | |||
250 | $attributes = []; |
||
251 | foreach ($this->attributes as $attribute => $attributeParams) { |
||
252 | if (empty($attributeParams['extractor'])) { |
||
253 | $extractorId = $attribute; |
||
254 | } else { |
||
255 | $extractorId = $attributeParams['extractor']; |
||
256 | } |
||
257 | |||
258 | if (empty($attributeParams['value']) && $this->importer->hasExtractor($extractorId)) { |
||
259 | if (empty($relationsModels[$rowNbr]) || empty($relationsModels[$rowNbr][$attribute])) { |
||
260 | // $attributes[$attribute] = null; |
||
261 | } else { |
||
262 | $p = $relationsModels[$rowNbr][$attribute]->id; |
||
263 | $attributes[$attribute] = (int)$p; |
||
264 | } |
||
265 | } else { |
||
266 | if (!empty($attributeParams['value'])) { |
||
267 | $attributes[$attribute] = $attributeParams['value']; |
||
268 | } else { |
||
269 | if (empty($attributeParams['column'])) { |
||
270 | throw new Exception('Not setted column for attribute ' . $attribute . ' for extractor ' . $this->id); |
||
271 | } |
||
272 | |||
273 | if (empty($row[$attributeParams['column'] - 1])) { |
||
274 | continue 2; |
||
275 | } else { |
||
276 | $attributes[$attribute] = $row[$attributeParams['column'] - 1]; |
||
277 | } |
||
278 | } |
||
279 | } |
||
280 | } |
||
281 | |||
282 | $attributesNames = $this->getAttributesNamesForFind(); |
||
283 | $searchedAttributes = []; |
||
284 | foreach ($attributesNames as $attributesName) { |
||
285 | $modelClass = $this->query->modelClass; |
||
286 | if (empty($attributes[$attributesName])) { |
||
287 | $value = null; |
||
288 | } else { |
||
289 | $value = $attributes[$attributesName]; |
||
290 | } |
||
291 | |||
292 | if (method_exists($modelClass, 'filtrateAttribute')) { |
||
293 | $value = $modelClass::filtrateAttribute($attributesName, $value); |
||
294 | $attributes[$attributesName] = $value; |
||
295 | } |
||
296 | |||
297 | $searchedAttributes[$attributesName] = $value; |
||
298 | } |
||
299 | |||
300 | $uniqueKey = serialize($searchedAttributes); |
||
301 | if (isset($this->modelsByUniqueKey[$uniqueKey])) { |
||
302 | $model = $this->modelsByUniqueKey[$uniqueKey]; |
||
303 | } else { |
||
304 | $model = new $this->query->modelClass; |
||
305 | $this->modelsByUniqueKey[$uniqueKey] = $model; |
||
306 | } |
||
307 | |||
308 | if ($this->isDelete && !$model->isNewRecord) { |
||
309 | unset($this->deletedIds[$model->id]); |
||
310 | } |
||
311 | |||
312 | $modelAttributes = $model->getAttributes(array_keys($attributes)); |
||
313 | if (!$isSave || $model->isNewRecord || array_diff($attributes, $modelAttributes)) { |
||
314 | $model->attributes = $attributes; |
||
315 | |||
316 | $models[$rowNbr] = $model; |
||
317 | } |
||
318 | |||
319 | // if ($model->isNewRecord && empty($this->isNoCreate)) { |
||
320 | // echo 1; |
||
321 | // } |
||
322 | } |
||
323 | |||
324 | $this->endOperation('models collect'); |
||
325 | |||
326 | $this->endOperation('extract'); |
||
327 | if ($isSave) { |
||
328 | foreach ($models as $rowNbr => $model) { |
||
329 | if (!$this->isBadRow($rowNbr)) { |
||
330 | $this->saveModel($model, $rowNbr); |
||
331 | } |
||
332 | } |
||
333 | } |
||
334 | |||
335 | return $models; |
||
336 | } |
||
337 | |||
338 | protected $times = []; |
||
339 | protected function startOperation($name) { |
||
340 | echo 'start ' . $name . ' ' . $this->id . "\n"; |
||
341 | $this->times[$name] = microtime(true); |
||
342 | } |
||
343 | |||
344 | protected function endOperation($name) { |
||
345 | $time = microtime(true) - $this->times[$name]; |
||
346 | echo 'end ' . $name . ' ' . $this->id . ' after ' . $time . ' seconds' . "\n"; |
||
347 | } |
||
348 | |||
349 | protected function triggerOperation($name) { |
||
350 | echo $name . ' ' . $this->id . "\n"; |
||
351 | } |
||
352 | |||
353 | protected function isBadRow($rowNbr) { |
||
354 | return $this->importer->isBadRow($rowNbr); |
||
355 | } |
||
356 | |||
357 | protected function logError($message, $rowNbr, $model, $columnNbr = null, $isMarkBad = true) { |
||
358 | if ($isMarkBad) { |
||
359 | $this->importer->setIsBadRow($rowNbr); |
||
360 | } |
||
361 | |||
362 | $this->importer->logError($message, $rowNbr, $model, $columnNbr); |
||
363 | } |
||
364 | |||
365 | /** |
||
366 | * @return array |
||
367 | */ |
||
368 | public function getAttributesNamesForFind(): array |
||
369 | { |
||
370 | $attributesNames = []; |
||
371 | foreach ($this->attributes as $attribute => $attributeParams) { |
||
372 | if (!empty($attributeParams['isFind'])) { |
||
373 | $attributesNames[] = $attribute; |
||
374 | } |
||
375 | } |
||
376 | return $attributesNames; |
||
377 | } |
||
378 | |||
379 | /** |
||
380 | * @param $model |
||
381 | * @return mixed |
||
382 | */ |
||
383 | protected function saveModel($model, $rowNbr) |
||
384 | { |
||
385 | $currentRowNbr = $this->importer->getCurrentStackRowNbr() + $rowNbr; |
||
386 | $modelString = $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)); |
||
387 | echo 'Row #' . $currentRowNbr . ': '; |
||
388 | if (!$model->validate()) { |
||
389 | $message = 'Error validate ' . $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)) . ' ' . serialize($model->errors); |
||
390 | $this->logError($message, $rowNbr, $model); |
||
391 | return false; |
||
392 | } |
||
393 | |||
394 | $isNewRecord = ($model->isNewRecord && !$this->isNoCreate); |
||
395 | $isUpdatedRecord = (!$model->isNewRecord && !$this->isNoUpdate && !empty($model->dirtyAttributes)); |
||
396 | if ($isNewRecord || $isUpdatedRecord) { |
||
397 | if ($isNewRecord) { |
||
398 | $reason = 'created'; |
||
399 | } else { |
||
400 | $reason = 'updated'; |
||
401 | } |
||
402 | |||
403 | echo 'Saving ' . $modelString . ' because they is ' . $reason . "\n"; |
||
404 | if ($isUpdatedRecord) { |
||
405 | echo 'Changed attributes ' . serialize(array_keys($model->dirtyAttributes)) . "\n"; |
||
406 | $oldValues = []; |
||
407 | foreach ($model->dirtyAttributes as $attribute => $value) { |
||
408 | $oldValues[$attribute] = $model->oldAttributes[$attribute]; |
||
409 | } |
||
410 | |||
411 | echo 'Old values ' . serialize($oldValues) . "\n"; |
||
412 | echo 'New values ' . serialize($model->dirtyAttributes) . "\n"; |
||
413 | } |
||
414 | |||
415 | if (!$model->save()) { |
||
416 | $this->logError('Error while saving ' . $modelString, $rowNbr, $model); |
||
417 | } |
||
418 | } else { |
||
419 | echo $modelString . ' Is skipped because is not changed' . "\n"; |
||
420 | } |
||
421 | |||
422 | return true; |
||
423 | } |
||
424 | } |