1 | <?php |
||
2 | /** |
||
3 | * Created by PhpStorm. |
||
4 | * User: execut |
||
5 | * Date: 12/22/17 |
||
6 | * Time: 12:57 PM |
||
7 | */ |
||
8 | |||
9 | namespace execut\import\components; |
||
10 | |||
11 | |||
12 | use execut\import\ModelInterface; |
||
13 | use execut\import\Query; |
||
14 | use yii\base\Component; |
||
15 | use yii\base\Exception; |
||
16 | use yii\db\ActiveQuery; |
||
17 | use yii\db\ActiveRecord; |
||
18 | use yii\helpers\ArrayHelper; |
||
19 | |||
20 | class ModelsExtractor extends Component |
||
21 | { |
||
22 | public $id = ''; |
||
23 | |||
24 | /** |
||
25 | * @var ActiveQuery $query |
||
26 | */ |
||
27 | public $query; |
||
28 | public $scopes = null; |
||
29 | public $isImport = false; |
||
30 | public $attributes = []; |
||
31 | /** |
||
32 | * @var Importer |
||
33 | */ |
||
34 | public $importer = null; |
||
35 | protected $modelsByUniqueKey = []; |
||
36 | public $isNoCreate = false; |
||
37 | public $isNoUpdate = false; |
||
38 | public $isDelete = false; |
||
39 | public $deletedIds = []; |
||
40 | public $uniqueKeys = null; |
||
41 | |||
42 | public function reset() { |
||
43 | $this->modelsByUniqueKey = []; |
||
44 | } |
||
45 | |||
46 | public function deleteOldRecords() { |
||
47 | if ($this->isDelete) { |
||
48 | $ids = $this->deletedIds; |
||
49 | if (!empty($ids)) { |
||
50 | if (count($ids) > 500000) { |
||
51 | throw new Exception('Many than 500 000 records to delete. Dangerous situation.'); |
||
52 | } |
||
53 | |||
54 | $modelClass = $this->query->modelClass; |
||
55 | while ($idsPart = array_splice($ids, 0, 65534)) { |
||
56 | if (count($idsPart) > 0) { |
||
57 | $modelClass::deleteAll([ |
||
58 | 'id' => $idsPart |
||
59 | ]); |
||
60 | } |
||
61 | } |
||
62 | } |
||
63 | } |
||
64 | } |
||
65 | |||
66 | public function getModels($isSave = true, $isMarkBad = true) { |
||
67 | /** |
||
68 | * @var ActiveRecord $model |
||
69 | */ |
||
70 | $this->startOperation('extract'); |
||
71 | $whereValues = []; |
||
72 | $relationsModels = []; |
||
73 | $this->startOperation('construct where'); |
||
74 | foreach ($this->attributes as $attribute => $attributeParams) { |
||
75 | if (empty($attributeParams['extractor'])) { |
||
76 | $extractorId = $attribute; |
||
77 | } else { |
||
78 | $extractorId = $attributeParams['extractor']; |
||
79 | } |
||
80 | |||
81 | if (empty($attributeParams['value']) && ($extractor = $this->importer->getExtractor($extractorId))) { |
||
82 | $models = $extractor->getModels(false, $isMarkBad && !empty($attributeParams['isFind'])); |
||
83 | foreach ($this->importer->rows as $rowNbr => $row) { |
||
84 | if ($this->isBadRow($rowNbr)) { |
||
85 | continue; |
||
86 | } |
||
87 | |||
88 | if (empty($models[$rowNbr])) { |
||
89 | if ($isMarkBad && !empty($attributeParams['isFind'])) { |
||
90 | $this->importer->setIsBadRow($rowNbr); |
||
91 | } |
||
92 | |||
93 | continue; |
||
94 | } |
||
95 | |||
96 | $model = $models[$rowNbr]; |
||
97 | |||
98 | if (empty($relationsModels[$rowNbr])) { |
||
99 | $relationsModels[$rowNbr] = []; |
||
100 | } |
||
101 | |||
102 | if (empty($extractor->isNoCreate) || !$model->isNewRecord) { |
||
103 | $relationsModels[$rowNbr][$attribute] = $model; |
||
104 | } |
||
105 | |||
106 | if (!empty($attributeParams['isFind'])) { |
||
107 | if (!$model->isNewRecord && !empty($model->dirtyAttributes)) { |
||
108 | if (empty($extractor->isNoUpdate)) { |
||
109 | if (!$this->saveModel($model, $rowNbr)) { |
||
110 | unset($whereValues[$rowNbr]); |
||
111 | continue; |
||
112 | } |
||
113 | } |
||
114 | } else if ($model->isNewRecord) { |
||
115 | if (empty($extractor->isNoCreate)) { |
||
116 | // Поиск не нужен, модель новая |
||
117 | unset($whereValues[$rowNbr]); |
||
118 | $this->saveModel($model, $rowNbr); |
||
119 | continue; |
||
120 | } |
||
121 | } |
||
122 | |||
123 | if ($model->isNewRecord && !empty($extractor->isNoCreate)) { |
||
124 | $this->logError('Related record is not founded with attributes ' . serialize(array_filter($model->attributes)), $rowNbr, $model, null, $isMarkBad); |
||
125 | unset($whereValues[$rowNbr]); |
||
126 | continue; |
||
127 | } |
||
128 | |||
129 | if (empty($whereValues[$rowNbr])) { |
||
130 | $whereValues[$rowNbr] = []; |
||
131 | } |
||
132 | |||
133 | $whereValues[$rowNbr][$attribute] = (int)$model->id; |
||
134 | } |
||
135 | } |
||
136 | } else { |
||
137 | if (!empty($attributeParams['isFind'])) { |
||
138 | foreach ($this->importer->rows as $rowNbr => $row) { |
||
139 | if ($this->isBadRow($rowNbr)) { |
||
140 | continue; |
||
141 | } |
||
142 | |||
143 | if (!empty($attributeParams['value'])) { |
||
144 | $whereValues[$rowNbr][$attribute] = $attributeParams['value']; |
||
145 | continue; |
||
146 | } |
||
147 | |||
148 | if (empty($attributeParams['column'])) { |
||
149 | return []; |
||
150 | throw new Exception('Column key is required for attribute params. Extractor: ' . $extractorId . '. Attribute params: ' . var_export($attributeParams, true)); |
||
0 ignored issues
–
show
|
|||
151 | } |
||
152 | |||
153 | if (empty($row[$attributeParams['column'] - 1])) { |
||
154 | $this->logError($attribute . ' is required for record find', $rowNbr, null, $attributeParams['column']); |
||
155 | unset($whereValues[$rowNbr]); |
||
156 | continue; |
||
157 | } |
||
158 | |||
159 | if (empty($whereValues[$rowNbr])) { |
||
160 | $whereValues[$rowNbr] = []; |
||
161 | } |
||
162 | |||
163 | $modelClass = $this->query->modelClass; |
||
164 | $value = $row[$attributeParams['column'] - 1]; |
||
165 | if (method_exists($modelClass, 'filtrateAttribute')) { |
||
166 | $value = $modelClass::filtrateAttribute($attribute, $value); |
||
167 | } |
||
168 | |||
169 | $whereValues[$rowNbr][$attribute] = $value; |
||
170 | } |
||
171 | } |
||
172 | } |
||
173 | } |
||
174 | |||
175 | $result = []; |
||
176 | foreach ($whereValues as $rowNbr => $whereValue) { |
||
177 | if ($this->importer->isBadRow($rowNbr)) { |
||
178 | continue; |
||
179 | } |
||
180 | |||
181 | $uniqueKey = serialize($whereValue); |
||
182 | if (!isset($this->modelsByUniqueKey[$uniqueKey])) { |
||
183 | $result[$uniqueKey] = $whereValue; |
||
184 | } |
||
185 | } |
||
186 | |||
187 | $whereValues = $result; |
||
188 | |||
189 | $this->endOperation('construct where'); |
||
190 | if (count($whereValues)) { |
||
191 | $attributesNames = $this->getAttributesNamesForFind(); |
||
192 | $query = clone $this->query; |
||
193 | $query->indexBy(function ($row) use ($attributesNames) { |
||
194 | $searchedAttributes = []; |
||
195 | foreach ($attributesNames as $attributesName) { |
||
196 | $modelClass = $this->query->modelClass; |
||
197 | $value = $row[$attributesName]; |
||
198 | if (method_exists($modelClass, 'filtrateAttribute')) { |
||
199 | $value = $modelClass::filtrateAttribute($attributesName, $value); |
||
200 | } |
||
201 | |||
202 | $searchedAttributes[$attributesName] = $value; |
||
203 | } |
||
204 | return serialize($searchedAttributes); |
||
205 | }); |
||
206 | |||
207 | if ($this->scopes !== null) { |
||
208 | foreach ($this->scopes as $scope) { |
||
209 | $scope($query, [ |
||
210 | 'IN', |
||
211 | $attributesNames, |
||
212 | $whereValues, |
||
213 | ]); |
||
214 | } |
||
215 | } else { |
||
216 | $query->andWhere([ |
||
217 | 'IN', |
||
218 | $attributesNames, |
||
219 | $whereValues, |
||
220 | ]); |
||
221 | } |
||
222 | |||
223 | $this->startOperation('find'); |
||
224 | $models = $query->all(); |
||
225 | $this->endOperation('find'); |
||
226 | $this->startOperation('keys collect'); |
||
227 | foreach ($models as $uniqueKey => $model) { |
||
228 | if ($this->uniqueKeys !== null) { |
||
229 | $callback = $this->uniqueKeys; |
||
230 | $uniqueKeys = $callback($model, $attributesNames, $whereValues); |
||
231 | } else { |
||
232 | $uniqueKeys = [$uniqueKey]; |
||
233 | } |
||
234 | |||
235 | foreach ($uniqueKeys as $uniqueKey) { |
||
236 | $this->modelsByUniqueKey[$uniqueKey] = $model; |
||
237 | } |
||
238 | } |
||
239 | |||
240 | $this->endOperation('keys collect'); |
||
241 | } |
||
242 | |||
243 | $models = []; |
||
244 | $this->startOperation('models collect'); |
||
245 | foreach ($this->importer->rows as $rowNbr => $row) { |
||
246 | if ($this->isBadRow($rowNbr)) { |
||
247 | continue; |
||
248 | } |
||
249 | |||
250 | $attributes = []; |
||
251 | foreach ($this->attributes as $attribute => $attributeParams) { |
||
252 | if (empty($attributeParams['extractor'])) { |
||
253 | $extractorId = $attribute; |
||
254 | } else { |
||
255 | $extractorId = $attributeParams['extractor']; |
||
256 | } |
||
257 | |||
258 | if (empty($attributeParams['value']) && $this->importer->hasExtractor($extractorId)) { |
||
259 | if (empty($relationsModels[$rowNbr]) || empty($relationsModels[$rowNbr][$attribute])) { |
||
260 | // $attributes[$attribute] = null; |
||
261 | } else { |
||
262 | $p = $relationsModels[$rowNbr][$attribute]->id; |
||
263 | $attributes[$attribute] = (int)$p; |
||
264 | } |
||
265 | } else { |
||
266 | if (!empty($attributeParams['value'])) { |
||
267 | $attributes[$attribute] = $attributeParams['value']; |
||
268 | } else { |
||
269 | if (empty($attributeParams['column'])) { |
||
270 | throw new Exception('Not setted column for attribute ' . $attribute . ' for extractor ' . $this->id); |
||
271 | } |
||
272 | |||
273 | if (empty($row[$attributeParams['column'] - 1])) { |
||
274 | continue 2; |
||
275 | } else { |
||
276 | $attributes[$attribute] = $row[$attributeParams['column'] - 1]; |
||
277 | } |
||
278 | } |
||
279 | } |
||
280 | } |
||
281 | |||
282 | $attributesNames = $this->getAttributesNamesForFind(); |
||
283 | $searchedAttributes = []; |
||
284 | foreach ($attributesNames as $attributesName) { |
||
285 | $modelClass = $this->query->modelClass; |
||
286 | if (empty($attributes[$attributesName])) { |
||
287 | $value = null; |
||
288 | } else { |
||
289 | $value = $attributes[$attributesName]; |
||
290 | } |
||
291 | |||
292 | if (method_exists($modelClass, 'filtrateAttribute')) { |
||
293 | $value = $modelClass::filtrateAttribute($attributesName, $value); |
||
294 | $attributes[$attributesName] = $value; |
||
295 | } |
||
296 | |||
297 | $searchedAttributes[$attributesName] = $value; |
||
298 | } |
||
299 | |||
300 | $uniqueKey = serialize($searchedAttributes); |
||
301 | if (isset($this->modelsByUniqueKey[$uniqueKey])) { |
||
302 | $model = $this->modelsByUniqueKey[$uniqueKey]; |
||
303 | } else { |
||
304 | $model = new $this->query->modelClass; |
||
305 | $this->modelsByUniqueKey[$uniqueKey] = $model; |
||
306 | } |
||
307 | |||
308 | if ($this->isDelete && !$model->isNewRecord) { |
||
309 | unset($this->deletedIds[$model->id]); |
||
310 | } |
||
311 | |||
312 | $modelAttributes = $model->getAttributes(array_keys($attributes)); |
||
313 | if (!$isSave || $model->isNewRecord || array_diff($attributes, $modelAttributes)) { |
||
314 | $model->attributes = $attributes; |
||
315 | |||
316 | $models[$rowNbr] = $model; |
||
317 | } |
||
318 | |||
319 | // if ($model->isNewRecord && empty($this->isNoCreate)) { |
||
320 | // echo 1; |
||
321 | // } |
||
322 | } |
||
323 | |||
324 | $this->endOperation('models collect'); |
||
325 | |||
326 | $this->endOperation('extract'); |
||
327 | if ($isSave) { |
||
328 | foreach ($models as $rowNbr => $model) { |
||
329 | if (!$this->isBadRow($rowNbr)) { |
||
330 | $this->saveModel($model, $rowNbr); |
||
331 | } |
||
332 | } |
||
333 | } |
||
334 | |||
335 | return $models; |
||
336 | } |
||
337 | |||
338 | protected $times = []; |
||
339 | protected function startOperation($name) { |
||
340 | echo 'start ' . $name . ' ' . $this->id . "\n"; |
||
341 | $this->times[$name] = microtime(true); |
||
342 | } |
||
343 | |||
344 | protected function endOperation($name) { |
||
345 | $time = microtime(true) - $this->times[$name]; |
||
346 | echo 'end ' . $name . ' ' . $this->id . ' after ' . $time . ' seconds' . "\n"; |
||
347 | } |
||
348 | |||
349 | protected function triggerOperation($name) { |
||
350 | echo $name . ' ' . $this->id . "\n"; |
||
351 | } |
||
352 | |||
353 | protected function isBadRow($rowNbr) { |
||
354 | return $this->importer->isBadRow($rowNbr); |
||
355 | } |
||
356 | |||
357 | protected function logError($message, $rowNbr, $model, $columnNbr = null, $isMarkBad = true) { |
||
358 | if ($isMarkBad) { |
||
359 | $this->importer->setIsBadRow($rowNbr); |
||
360 | } |
||
361 | |||
362 | $this->importer->logError($message, $rowNbr, $model, $columnNbr); |
||
363 | } |
||
364 | |||
365 | /** |
||
366 | * @return array |
||
367 | */ |
||
368 | public function getAttributesNamesForFind(): array |
||
369 | { |
||
370 | $attributesNames = []; |
||
371 | foreach ($this->attributes as $attribute => $attributeParams) { |
||
372 | if (!empty($attributeParams['isFind'])) { |
||
373 | $attributesNames[] = $attribute; |
||
374 | } |
||
375 | } |
||
376 | return $attributesNames; |
||
377 | } |
||
378 | |||
379 | /** |
||
380 | * @param $model |
||
381 | * @return mixed |
||
382 | */ |
||
383 | protected function saveModel($model, $rowNbr) |
||
384 | { |
||
385 | $currentRowNbr = $this->importer->getCurrentStackRowNbr() + $rowNbr; |
||
386 | $modelString = $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)); |
||
387 | echo 'Row #' . $currentRowNbr . ': '; |
||
388 | if (!$model->validate()) { |
||
389 | $message = 'Error validate ' . $model->tableName() . ' #' . $model->id . ' ' . serialize(array_filter($model->attributes)) . ' ' . serialize($model->errors); |
||
390 | $this->logError($message, $rowNbr, $model); |
||
391 | return false; |
||
392 | } |
||
393 | |||
394 | $isNewRecord = ($model->isNewRecord && !$this->isNoCreate); |
||
395 | $isUpdatedRecord = (!$model->isNewRecord && !$this->isNoUpdate && !empty($model->dirtyAttributes)); |
||
396 | if ($isNewRecord || $isUpdatedRecord) { |
||
397 | if ($isNewRecord) { |
||
398 | $reason = 'created'; |
||
399 | } else { |
||
400 | $reason = 'updated'; |
||
401 | } |
||
402 | |||
403 | echo 'Saving ' . $modelString . ' because they is ' . $reason . "\n"; |
||
404 | if ($isUpdatedRecord) { |
||
405 | echo 'Changed attributes ' . serialize(array_keys($model->dirtyAttributes)) . "\n"; |
||
406 | $oldValues = []; |
||
407 | foreach ($model->dirtyAttributes as $attribute => $value) { |
||
408 | $oldValues[$attribute] = $model->oldAttributes[$attribute]; |
||
409 | } |
||
410 | |||
411 | echo 'Old values ' . serialize($oldValues) . "\n"; |
||
412 | echo 'New values ' . serialize($model->dirtyAttributes) . "\n"; |
||
413 | } |
||
414 | |||
415 | if (!$model->save()) { |
||
416 | $this->logError('Error while saving ' . $modelString, $rowNbr, $model); |
||
417 | } |
||
418 | } else { |
||
419 | echo $modelString . ' Is skipped because is not changed' . "\n"; |
||
420 | } |
||
421 | |||
422 | return true; |
||
423 | } |
||
424 | } |
This check looks for unreachable code. It uses sophisticated control flow analysis techniques to find statements which will never be executed.
Unreachable code is most often the result of
return
,die
orexit
statements that have been added for debug purposes.In the above example, the last
return false
will never be executed, because a return statement has already been met in every possible execution path.