ConsoleController::actionCheckSource() - Code Metrics - execut/yii2-import - Measure and Improve Code Quality continuously with Scrutinizer

ConsoleController::actionCheckSource() B
last analyzed 2021-03-09 12:54 UTC

↳ Parent: ConsoleController

Complexity

Conditions	8
Paths	20

Size

Total Lines	59
Code Lines	37

Duplication

Lines	0
Ratio	0 %

Importance

Changes	3
Bugs	0	Features	2

Metric	Value
eloc	37
c	3
b	0
f	2
dl	0
loc	59
rs	8.0835
cc	8
nc	20
nop	3

How to fix Long Method

<?php
/**
 * User: execut
 * Date: 18.07.16
 * Time: 10:14
 */

namespace execut\import\controllers;


use execut\crudFields\fields\Field;
use execut\import\components\Importer;
use execut\import\components\parser\exception\Exception;
use execut\import\components\parser\Stack;
use execut\import\models\FilesSource;
use execut\import\models\FilesStatuse;
use execut\import\models\File;
use execut\import\models\Log;
use execut\import\models\Setting;

use yii\console\Controller;
use yii\log\Logger;
use yii\mutex\Mutex;

class ConsoleController extends Controller
{
    public $loadsLimit = 20;
    public $stackSize = 650;
    public $offset = 0;
    public $fileId = null;
    protected $lastCheckedRow = 0;

    public function options($actionID)
    {
        if ($actionID === 'index') {
            return [
                'fileId',
                'stackSize',
                'offset',
            ];
        }
        // $actionId might be used in subclasses to provide options specific to action id
        return ['color', 'interactive', 'help'];
    }

    public function actionIndex() {
        ini_set('memory_limit', -1);
        $id = $this->fileId;
//        $this->clearOldFiles();
        $q = File::find();
        if (YII_ENV !== 'dev') {

            $this->markErrorFiles();
        }

        if ($id === null) {
            $q->byHostName(gethostname());
            $q
                ->isForImport()
                ->isNotLoadedOrStop()
                ->isOnlyFresh()
                ->orderBy('created ASC');
            $currentFilesCount = File::find()->byHostName(gethostname())->isLoading()->count();
            if ($currentFilesCount >= $this->loadsLimit) {
                echo 'Files limit reached ' . $this->loadsLimit . '. Now loaded ' . $currentFilesCount . ' files';
                return;
            }
        } else {
            $q->byId($id);
        }

        while (true) {
            if ($id === null) {
                $this->waitForRelease();
            }

            /**
             * @var File $file
             */
            $file = $q->one();
            if ($id === null && !$file) {
                $this->release();
                break;
            }

            $file->triggerLoading();
            if ($id === null) {
                $this->release();
            }

            $this->deleteOldFilesBySetting($file);

            $this->parseFile($file);
            if ($id !== null) {
                break;
            }
        }
    }

    public function actionReleaseTrigger() {
        $this->release();
    }

    protected function deleteOldFilesBySetting($file) {
        $this->waitForRelease();
        $q = File::find()->byImportSettingId($file->import_setting_id)->andWhere([
            '<>',
            'id',
            $file->id,
        ])->select('id');
        $files = $q->all();
        foreach ($files as $file) {

            if ($file->delete()) {
                echo 'Deleted file #' . $file->id . "\n";

            }
        }

        $this->release();
    }

    protected function markErrorFiles()
    {
        $this->waitForRelease();
        $this->stdout('Start check failed files' . "\n");
        /**
         * @var File $file
         */
        $files = File::find()->byHostName(gethostname())->isWithoutProcess()->isInProgress()->all();
        foreach ($files as $file) {
            $this->stdout('File ' . $file->id . ' start mark as error' . "\n");

            $attributes = [
                'level' => Logger::LEVEL_ERROR,
                'category' => 'import.notFoundProcess',
                'message' => 'The process ' . $file->process_id . ' to import the file was not found',

            ];
            $file->logError($attributes);
            $file->triggerException();
            $this->stdout('File ' . $file->id . ' is marked as errored' . "\n");
        }

        $this->release();
        $this->stdout('End check failed files' . "\n");
    }

    protected function clearOldFiles()
    {
        while (true) {
            $this->waitForRelease();
            /**
             * @var File $file
             */
            $file = File::find()->byHostName(gethostname())->isForClean()->one();
            if (!$file) {
                $this->release();
                break;
            }

            $file->triggerDeleting();
            $file->delete();
            $this->release();
        }
    }

    protected function parseFile(File $file) {
        $this->stdout('Start parse file #' . $file->id . ' ' . $file->name . "\n");
        try {
            $file->scenario = 'import';
            $data = $file->getRows();
            if ($this->offset > 0) {
                $data = array_splice($data, $this->offset);
            }
    //        $data = array_splice($data, 23400, 1000000);
            ini_set('error_reporting', E_ERROR);
            $file->rows_count = count($data);
            $file->save();
            ini_set('error_reporting', E_ALL);
        } catch (\Exception $e) {
            $attributes = [
                'level' => Logger::LEVEL_ERROR,
                'category' => 'import.fatalError',
                'message' => $e->getMessage() . "\n" . $e->getTraceAsString(),
            ];
            $file->logError($attributes);
            $file->triggerException();
            throw $e;
        }

        $stacksSettings = $file->getSettings();
        //        \yii::$app->db->close();
        $importer = new Importer([
            'file' => $file,
            'settings' => $stacksSettings,
            'data' => $data,
            'stackSize' => $this->stackSize,
        ]);
        $importer->run();
    }

    public function actionCheckSource($type = 'email', $id = null, $isIgnoreMutex = false) {
        imap_timeout( IMAP_CLOSETIMEOUT, 60);
        /**
         * @var Mutex $mutex
         */
        $mutex = \yii::$app->mutex;
        $mutexKey = self::class . '_' . $type;
        while (!$isIgnoreMutex && !$mutex->acquire($mutexKey)) {
            sleep(1);
        }

        $this->stdout('Checking source type ' . $type . "\n");
        $q = Setting::find();
        if ($id) {
            $q->andWhere(['id' => $id]);
        } else {
            $q->byImportFilesSource_key($type);
        }

        /**
         * @var Setting[] $importSettings
         */
        $importSettings = $q->all();
        FilesSource::$emailAdapter = null;
        foreach ($importSettings as $setting) {
            $source = $setting->getSource();
            $files = $source->getFiles();
            if (!empty($files)) {
                foreach ($files as $file) {
                    $md5 = md5_file($file->filePath);
                    if ($importFile = File::find()->byMd5($md5)->select(['id', 'updated'])->one()) {
                        $importFile->updated = date('Y-m-d H:i:s');
                        /**
                         * @var File $file
                         */
                        $importFile->save(false, [
                            'updated'
                        ]);
                        echo 'File with md5 ' . $md5 . ' is already exists' . "\n";
                    } else {
                        $importFile = new File();
                        $importFile->scenario = Field::SCENARIO_FORM;
                        $fileInfo = pathinfo($file->filePath);
                        $importFile->attributes = [
                            'name' => $file->fileName,
                            'extension' => $fileInfo['extension'],
                            'mime_type' => mime_content_type($file->filePath),
                            'import_setting_id' => $setting->id,
                            'import_files_source_id' => $setting->filesSource->id,

                            'content' => $file->content,
                        ];

                        $this->saveModel($importFile);
                    }
                }
            }
        }

        $mutex->release($mutexKey);
    }

    public function actionCheckSourceDaemon($type = 'email') {
        while (true) {
            $this->actionCheckSource($type);
            sleep(60);
        }
    }

    protected function saveModel($model) {
        if ($model->save()) {
            $this->stdout('Model ' . $model . ' is saved' . "\n");
        } else {
            $this->stderr('Model ' . $model . ' is errors: ' . var_export($model->errors, true) . "\n");
        }
    }

    /**
     * @return Mutex
     */
    protected function waitForRelease(): Mutex
    {
        /**
         * @var Mutex $mutex
         */
        $mutex = \yii::$app->mutex;
        while (!$mutex->acquire(__CLASS__)) {
            echo 'Wait for release' . "\n";
            sleep(1);
        }

        return $mutex;
    }

    protected function release(): void
    {
        /**
         * @var Mutex $mutex
         */
        $mutex = \yii::$app->mutex;
        $mutex->release(__CLASS__);
    }
}

1			<?php
2			/**
3			* User: execut
4			* Date: 18.07.16
5			* Time: 10:14
6			*/
7
8			namespace execut\import\controllers;
9
10
11			use execut\crudFields\fields\Field;
12			use execut\import\components\Importer;
13			use execut\import\components\parser\exception\Exception;
14			use execut\import\components\parser\Stack;
15			use execut\import\models\FilesSource;
16			use execut\import\models\FilesStatuse;
17			use execut\import\models\File;
18			use execut\import\models\Log;
19			use execut\import\models\Setting;
20
21			use yii\console\Controller;
22			use yii\log\Logger;
23			use yii\mutex\Mutex;
24
25			class ConsoleController extends Controller
26			{
27			public $loadsLimit = 20;
28			public $stackSize = 650;
29			public $offset = 0;
30			public $fileId = null;
31			protected $lastCheckedRow = 0;
32
33			public function options($actionID)
34			{
35			if ($actionID === 'index') {
36			return [
37			'fileId',
38			'stackSize',
39			'offset',
40			];
41			}
42			// $actionId might be used in subclasses to provide options specific to action id
43			return ['color', 'interactive', 'help'];
44			}
45
46			public function actionIndex() {
47			ini_set('memory_limit', -1);
48			$id = $this->fileId;
49			// $this->clearOldFiles();
50			$q = File::find();
51			if (YII_ENV !== 'dev') {
			0 ignored issues – show introduced 2019-07-22 07:56 UTC by Report Bug Copy Issue Report The condition `execut\import\controllers\YII_ENV !== 'dev'` is always `true`. Loading history...
52			$this->markErrorFiles();
53			}
54
55			if ($id === null) {
56			$q->byHostName(gethostname());
57			$q
58			->isForImport()
59			->isNotLoadedOrStop()
60			->isOnlyFresh()
61			->orderBy('created ASC');
62			$currentFilesCount = File::find()->byHostName(gethostname())->isLoading()->count();
63			if ($currentFilesCount >= $this->loadsLimit) {
64			echo 'Files limit reached ' . $this->loadsLimit . '. Now loaded ' . $currentFilesCount . ' files';
65			return;
66			}
67			} else {
68			$q->byId($id);
69			}
70
71			while (true) {
72			if ($id === null) {
73			$this->waitForRelease();
74			}
75
76			/**
77			* @var File $file
78			*/
79			$file = $q->one();
80			if ($id === null && !$file) {
81			$this->release();
82			break;
83			}
84
85			$file->triggerLoading();
86			if ($id === null) {
87			$this->release();
88			}
89
90			$this->deleteOldFilesBySetting($file);
91
92			$this->parseFile($file);
93			if ($id !== null) {
94			break;
95			}
96			}
97			}
98
99			public function actionReleaseTrigger() {
100			$this->release();
101			}
102
103			protected function deleteOldFilesBySetting($file) {
104			$this->waitForRelease();
105			$q = File::find()->byImportSettingId($file->import_setting_id)->andWhere([
106			'<>',
107			'id',
108			$file->id,
109			])->select('id');
110			$files = $q->all();
111			foreach ($files as $file) {
			0 ignored issues – show introduced 2021-03-09 12:54 UTC by Report Bug Copy Issue Report `$file` is overwriting one of the parameters of this function. Loading history...
112			if ($file->delete()) {
113			echo 'Deleted file #' . $file->id . "\n";
			0 ignored issues – show Bug introduced 2019-10-23 07:10 UTC by Report Bug Copy Issue Report Accessing `id` on the interface `yii\db\ActiveRecordInterface` suggest that you code against a concrete implementation. How about adding an `instanceof` check? Loading history...
114			}
115			}
116
117			$this->release();
118			}
119
120			protected function markErrorFiles()
121			{
122			$this->waitForRelease();
123			$this->stdout('Start check failed files' . "\n");
124			/**
125			* @var File $file
126			*/
127			$files = File::find()->byHostName(gethostname())->isWithoutProcess()->isInProgress()->all();
128			foreach ($files as $file) {
129			$this->stdout('File ' . $file->id . ' start mark as error' . "\n");
			0 ignored issues – show Bug introduced 2021-03-09 12:54 UTC by Report Bug Copy Issue Report Accessing `id` on the interface `yii\db\ActiveRecordInterface` suggest that you code against a concrete implementation. How about adding an `instanceof` check? Loading history...
130			$attributes = [
131			'level' => Logger::LEVEL_ERROR,
132			'category' => 'import.notFoundProcess',
133			'message' => 'The process ' . $file->process_id . ' to import the file was not found',
			0 ignored issues – show Bug introduced 2019-07-18 08:19 UTC by Report Bug Copy Issue Report Accessing `process_id` on the interface `yii\db\ActiveRecordInterface` suggest that you code against a concrete implementation. How about adding an `instanceof` check? Loading history...
134			];
135			$file->logError($attributes);
136			$file->triggerException();
137			$this->stdout('File ' . $file->id . ' is marked as errored' . "\n");
138			}
139
140			$this->release();
141			$this->stdout('End check failed files' . "\n");
142			}
143
144			protected function clearOldFiles()
145			{
146			while (true) {
147			$this->waitForRelease();
148			/**
149			* @var File $file
150			*/
151			$file = File::find()->byHostName(gethostname())->isForClean()->one();
152			if (!$file) {
153			$this->release();
154			break;
155			}
156
157			$file->triggerDeleting();
158			$file->delete();
159			$this->release();
160			}
161			}
162
163			protected function parseFile(File $file) {
164			$this->stdout('Start parse file #' . $file->id . ' ' . $file->name . "\n");
165			try {
166			$file->scenario = 'import';
167			$data = $file->getRows();
168			if ($this->offset > 0) {
169			$data = array_splice($data, $this->offset);
170			}
171			// $data = array_splice($data, 23400, 1000000);
172			ini_set('error_reporting', E_ERROR);
173			$file->rows_count = count($data);
174			$file->save();
175			ini_set('error_reporting', E_ALL);
176			} catch (\Exception $e) {
177			$attributes = [
178			'level' => Logger::LEVEL_ERROR,
179			'category' => 'import.fatalError',
180			'message' => $e->getMessage() . "\n" . $e->getTraceAsString(),
181			];
182			$file->logError($attributes);
183			$file->triggerException();
184			throw $e;
185			}
186
187			$stacksSettings = $file->getSettings();
188			// \yii::$app->db->close();
189			$importer = new Importer([
190			'file' => $file,
191			'settings' => $stacksSettings,
192			'data' => $data,
193			'stackSize' => $this->stackSize,
194			]);
195			$importer->run();
196			}
197
198			public function actionCheckSource($type = 'email', $id = null, $isIgnoreMutex = false) {
199			imap_timeout( IMAP_CLOSETIMEOUT, 60);
200			/**
201			* @var Mutex $mutex
202			*/
203			$mutex = \yii::$app->mutex;
204			$mutexKey = self::class . '_' . $type;
205			while (!$isIgnoreMutex && !$mutex->acquire($mutexKey)) {
206			sleep(1);
207			}
208
209			$this->stdout('Checking source type ' . $type . "\n");
210			$q = Setting::find();
211			if ($id) {
212			$q->andWhere(['id' => $id]);
213			} else {
214			$q->byImportFilesSource_key($type);
215			}
216
217			/**
218			* @var Setting[] $importSettings
219			*/
220			$importSettings = $q->all();
221			FilesSource::$emailAdapter = null;
222			foreach ($importSettings as $setting) {
223			$source = $setting->getSource();
224			$files = $source->getFiles();
225			if (!empty($files)) {
226			foreach ($files as $file) {
227			$md5 = md5_file($file->filePath);
228			if ($importFile = File::find()->byMd5($md5)->select(['id', 'updated'])->one()) {
229			$importFile->updated = date('Y-m-d H:i:s');
230			/**
231			* @var File $file
232			*/
233			$importFile->save(false, [
234			'updated'
235			]);
236			echo 'File with md5 ' . $md5 . ' is already exists' . "\n";
237			} else {
238			$importFile = new File();
239			$importFile->scenario = Field::SCENARIO_FORM;
240			$fileInfo = pathinfo($file->filePath);
241			$importFile->attributes = [
242			'name' => $file->fileName,
243			'extension' => $fileInfo['extension'],
244			'mime_type' => mime_content_type($file->filePath),
245			'import_setting_id' => $setting->id,
246			'import_files_source_id' => $setting->filesSource->id,
			0 ignored issues – show Bug Best Practice introduced 2019-07-18 08:19 UTC by Report Bug Copy Issue Report The property `filesSource` does not exist on `execut\import\models\Setting`. Since you implemented `__get`, consider adding a @property annotation. Loading history...
247			'content' => $file->content,
248			];
249
250			$this->saveModel($importFile);
251			}
252			}
253			}
254			}
255
256			$mutex->release($mutexKey);
257			}
258
259			public function actionCheckSourceDaemon($type = 'email') {
260			while (true) {
261			$this->actionCheckSource($type);
262			sleep(60);
263			}
264			}
265
266			protected function saveModel($model) {
267			if ($model->save()) {
268			$this->stdout('Model ' . $model . ' is saved' . "\n");
269			} else {
270			$this->stderr('Model ' . $model . ' is errors: ' . var_export($model->errors, true) . "\n");
271			}
272			}
273
274			/**
275			* @return Mutex
276			*/
277			protected function waitForRelease(): Mutex
278			{
279			/**
280			* @var Mutex $mutex
281			*/
282			$mutex = \yii::$app->mutex;
283			while (!$mutex->acquire(__CLASS__)) {
284			echo 'Wait for release' . "\n";
285			sleep(1);
286			}
287
288			return $mutex;
289			}
290
291			protected function release(): void
292			{
293			/**
294			* @var Mutex $mutex
295			*/
296			$mutex = \yii::$app->mutex;
297			$mutex->release(__CLASS__);
298			}
299			}

execut / yii2-import

ConsoleController::actionCheckSource() B last analyzed 2021-03-09 12:54 UTC

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like

ConsoleController::actionCheckSource() B
last analyzed 2021-03-09 12:54 UTC