StaleImagesRemovalTask::staleImagesRemovalForUser() - Code Metrics - Inspection of "Implement a FileService where to handle common cod..." - matiasdelellis/facerecognition - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — shared-storage-experiments ( 606554...2addb2 )

by Matias

created 2019-12-08 14:26 UTC

staleImagesRemovalForUser() B

↳ Parent: StaleImagesRemovalTask

Complexity

Conditions	8
Paths	9

Size

Total Lines	67
Code Lines	38

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	0
CRAP Score	72

Importance

Changes	1
Bugs	1	Features	0

Metric	Value
cc	8
eloc	38
c	1
b	1
f	0
nc	9
nop	2
dl	0
loc	67
ccs	0
cts	40
cp	0
crap	72
rs	8.0675

How to fix Long Method

<?php
/**
 * @copyright Copyright (c) 2017, Matias De lellis <[email protected]>
 * @copyright Copyright (c) 2018, Branko Kokanovic <[email protected]>
 *
 * @author Branko Kokanovic <[email protected]>
 *
 * @license GNU AGPL version 3 or any later version
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
namespace OCA\FaceRecognition\BackgroundJob\Tasks;

use OCP\IConfig;
use OCP\IUser;

use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\Node;

use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask;
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext;
use OCA\FaceRecognition\Db\Image;
use OCA\FaceRecognition\Db\ImageMapper;
use OCA\FaceRecognition\Db\FaceMapper;
use OCA\FaceRecognition\Db\PersonMapper;
use OCA\FaceRecognition\Migration\AddDefaultFaceModel;
use OCA\FaceRecognition\Service\FileService;

/**
 * Task that, for each user, crawls for all images in database,
 * checks if they actually exist and removes them if they don't.
 * It should be executed rarely.
 */
class StaleImagesRemovalTask extends FaceRecognitionBackgroundTask {
	const STALE_IMAGES_REMOVAL_NEEDED_KEY = "stale_images_removal_needed";
	const STALE_IMAGES_LAST_CHECKED_KEY = "stale_images_last_checked";

	/** @var IConfig Config */
	private $config;

	/** @var ImageMapper Image mapper */
	private $imageMapper;

	/** @var FaceMapper Face mapper */
	private $faceMapper;

	/** @var PersonMapper Person mapper */
	private $personMapper;

	/** @var FileService */
	private $fileService;

	/**
	 * @param IConfig $config Config
	 * @param ImageMapper $imageMapper Image mapper
	 * @param FaceMapper $faceMapper Face mapper
	 * @param PersonMapper $personMapper Person mapper
	 * @param FileService $fileService File Service
	 */
	public function __construct(IConfig      $config,
	                            ImageMapper  $imageMapper,
	                            FaceMapper   $faceMapper,
	                            PersonMapper $personMapper,
	                            FileService  $fileService) {
		parent::__construct();
		$this->config = $config;
		$this->imageMapper  = $imageMapper;
		$this->faceMapper   = $faceMapper;
		$this->personMapper = $personMapper;
		$this->fileService  = $fileService;
	}

	/**
	 * @inheritdoc
	 */
	public function description() {
		return "Crawl for stale images (either missing in filesystem or under .nomedia) and remove them from DB";
	}

	/**
	 * @inheritdoc
	 */
	public function execute(FaceRecognitionContext $context) {
		$this->setContext($context);

		$model = intval($this->config->getAppValue('facerecognition', 'model', AddDefaultFaceModel::DEFAULT_FACE_MODEL_ID));

		// Check if we are called for one user only, or for all user in instance.
		$staleRemovedImages = 0;
		$eligable_users = array();
		if (is_null($this->context->user)) {
			$this->context->userManager->callForSeenUsers(function (IUser $user) use (&$eligable_users) {
				$eligable_users[] = $user->getUID();
			});
		} else {
			$eligable_users[] = $this->context->user->getUID();
		}

		foreach($eligable_users as $user) {
			$staleImagesRemovalNeeded = $this->config->getUserValue(
				$user, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_REMOVAL_NEEDED_KEY, 'false');
			if ($staleImagesRemovalNeeded === 'false') {
				// Completely skip this task for this user, seems that we already did full scan for him
				$this->logDebug(sprintf('Skipping stale images removal for user %s as there is no need for it', $user));
				continue;
			}

			// Since method below can take long time, it is generator itself
			$generator = $this->staleImagesRemovalForUser($user, $model);
			foreach ($generator as $_) {
				yield;
			}
			$staleRemovedImages += $generator->getReturn();

			$this->config->setUserValue($user, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_REMOVAL_NEEDED_KEY, 'false');
			yield;
		}

		$this->context->propertyBag['StaleImagesRemovalTask_staleRemovedImages'] = $staleRemovedImages;
		return true;
	}

	/**
	 * Gets all images in database for a given user. For each image, check if it
	 * actually present in filesystem (and there is no .nomedia for it) and removes
	 * it from database if it is not present.
	 *
	 * @param string $userId ID of the user for which to remove stale images for
	 * @param int $model Used model
	 * @return \Generator|int Returns generator during yielding and finally returns int,
	 * which represent number of stale images removed
	 */
	private function staleImagesRemovalForUser(string $userId, int $model) {

		$this->fileService->setupFS($userId);

		$this->logDebug(sprintf('Getting all images for user %s', $userId));
		$allImages = $this->imageMapper->findImages($userId, $model);
		$this->logDebug(sprintf('Found %d images for user %s', count($allImages), $userId));
		yield;

		// Find if we stopped somewhere abruptly before. If we are, we need to start from that point.
		// If there is value, we start from beggining. Important is that:
		// * There needs to be some (any!) ordering here, we used "id" for ordering key
		// * New images will be processed, or some might be checked more than once, and that is OK
		//   Important part is that we make continuous progess.
		$lastChecked = intval($this->config->getUserValue(
			$userId, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_LAST_CHECKED_KEY, '0'));
		$this->logDebug(sprintf('Last checked image id for user %s is %d', $userId, $lastChecked));
		yield;

		// Now filter by those above last checked and sort remaining images
		$allImages = array_filter($allImages, function ($i) use($lastChecked) {
			return $i->id > $lastChecked;
		});
		usort($allImages, function ($i1, $i2) {
			return $i1->id <=> $i2->id;
		});
		$this->logDebug(sprintf(
			'After filtering and sorting, there is %d remaining stale images to check for user %s',
			count($allImages), $userId));
		yield;

		$handleSharedFiles = $this->config->getAppValue('facerecognition', 'handle-shared-files', 'false');

		// Now iterate and check remaining images
		$processed = 0;
		$imagesRemoved = 0;
		foreach ($allImages as $image) {
			// Try to get the file to ensure that exist.
			try {
				$file = $this->fileService->getFileById($image->getFile(), $userId);
			} catch (\OCP\Files\NotFoundException $e) {
				$file = null;
			}

			// Delete image doesn't exist anymore in filesystem or it is under .nomedia
			if (($file === null) ||
			    ($this->fileService->isUnderNoMedia($file)) ||
			    ($this->fileService->isSharedFile($file) && $handleSharedFiles !== 'true')) {
				$this->deleteImage($image, $userId);
				$imagesRemoved++;
			}

			// Remember last processed image
			$this->config->setUserValue(
				$userId, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_LAST_CHECKED_KEY, $image->id);

			// Yield from time to time
			$processed++;
			if ($processed % 10 === 0) {
				$this->logDebug(sprintf('Processed %d/%d stale images for user %s', $processed, count($allImages), $userId));
				yield;
			}
		}

		// Remove this value when we are done, so next cleanup can start from 0
		$this->config->deleteUserValue($userId, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_LAST_CHECKED_KEY);
		return $imagesRemoved;
	}

	private function deleteImage(Image $image, string $userId) {
		$this->logInfo(sprintf('Removing stale image %d for user %s', $image->id, $userId));
		// note that invalidatePersons depends on existence of faces for a given image,
		// and we must invalidate before we delete faces!
		// TODO: this is same method as in Watcher, find where to unify them.
		$this->personMapper->invalidatePersons($image->id);
		$this->faceMapper->removeFaces($image->id);
		$this->imageMapper->delete($image);
	}
}


1			<?php
2			/**
3			* @copyright Copyright (c) 2017, Matias De lellis <[email protected]>
4			* @copyright Copyright (c) 2018, Branko Kokanovic <[email protected]>
5			*
6			* @author Branko Kokanovic <[email protected]>
7			*
8			* @license GNU AGPL version 3 or any later version
9			*
10			* This program is free software: you can redistribute it and/or modify
11			* it under the terms of the GNU Affero General Public License as
12			* published by the Free Software Foundation, either version 3 of the
13			* License, or (at your option) any later version.
14			*
15			* This program is distributed in the hope that it will be useful,
16			* but WITHOUT ANY WARRANTY; without even the implied warranty of
17			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18			* GNU Affero General Public License for more details.
19			*
20			* You should have received a copy of the GNU Affero General Public License
21			* along with this program. If not, see <http://www.gnu.org/licenses/>.
22			*
23			*/
24			namespace OCA\FaceRecognition\BackgroundJob\Tasks;
25
26			use OCP\IConfig;
27			use OCP\IUser;
28
29			use OCP\Files\File;
30			use OCP\Files\Folder;
31			use OCP\Files\Node;
32
33			use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask;
34			use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext;
35			use OCA\FaceRecognition\Db\Image;
36			use OCA\FaceRecognition\Db\ImageMapper;
37			use OCA\FaceRecognition\Db\FaceMapper;
38			use OCA\FaceRecognition\Db\PersonMapper;
39			use OCA\FaceRecognition\Migration\AddDefaultFaceModel;
40			use OCA\FaceRecognition\Service\FileService;
41
42			/**
43			* Task that, for each user, crawls for all images in database,
44			* checks if they actually exist and removes them if they don't.
45			* It should be executed rarely.
46			*/
47			class StaleImagesRemovalTask extends FaceRecognitionBackgroundTask {
48			const STALE_IMAGES_REMOVAL_NEEDED_KEY = "stale_images_removal_needed";
49			const STALE_IMAGES_LAST_CHECKED_KEY = "stale_images_last_checked";
50
51			/** @var IConfig Config */
52			private $config;
53
54			/** @var ImageMapper Image mapper */
55			private $imageMapper;
56
57			/** @var FaceMapper Face mapper */
58			private $faceMapper;
59
60			/** @var PersonMapper Person mapper */
61			private $personMapper;
62
63			/** @var FileService */
64			private $fileService;
65
66			/**
67			* @param IConfig $config Config
68			* @param ImageMapper $imageMapper Image mapper
69			* @param FaceMapper $faceMapper Face mapper
70			* @param PersonMapper $personMapper Person mapper
71			* @param FileService $fileService File Service
72			*/
73			public function __construct(IConfig $config,
74			ImageMapper $imageMapper,
75			FaceMapper $faceMapper,
76			PersonMapper $personMapper,
77			FileService $fileService) {
78			parent::__construct();
79			$this->config = $config;
80			$this->imageMapper = $imageMapper;
81			$this->faceMapper = $faceMapper;
82			$this->personMapper = $personMapper;
83			$this->fileService = $fileService;
84			}
85
86			/**
87			* @inheritdoc
88			*/
89			public function description() {
90			return "Crawl for stale images (either missing in filesystem or under .nomedia) and remove them from DB";
91			}
92
93			/**
94			* @inheritdoc
95			*/
96			public function execute(FaceRecognitionContext $context) {
97			$this->setContext($context);
98
99			$model = intval($this->config->getAppValue('facerecognition', 'model', AddDefaultFaceModel::DEFAULT_FACE_MODEL_ID));
100
101			// Check if we are called for one user only, or for all user in instance.
102			$staleRemovedImages = 0;
103			$eligable_users = array();
104			if (is_null($this->context->user)) {
105			$this->context->userManager->callForSeenUsers(function (IUser $user) use (&$eligable_users) {
106			$eligable_users[] = $user->getUID();
107			});
108			} else {
109			$eligable_users[] = $this->context->user->getUID();
110			}
111
112			foreach($eligable_users as $user) {
113			$staleImagesRemovalNeeded = $this->config->getUserValue(
114			$user, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_REMOVAL_NEEDED_KEY, 'false');
115			if ($staleImagesRemovalNeeded === 'false') {
116			// Completely skip this task for this user, seems that we already did full scan for him
117			$this->logDebug(sprintf('Skipping stale images removal for user %s as there is no need for it', $user));
118			continue;
119			}
120
121			// Since method below can take long time, it is generator itself
122			$generator = $this->staleImagesRemovalForUser($user, $model);
123			foreach ($generator as $_) {
124			yield;
125			}
126			$staleRemovedImages += $generator->getReturn();
127
128			$this->config->setUserValue($user, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_REMOVAL_NEEDED_KEY, 'false');
129			yield;
130			}
131
132			$this->context->propertyBag['StaleImagesRemovalTask_staleRemovedImages'] = $staleRemovedImages;
133			return true;
134			}
135
136			/**
137			* Gets all images in database for a given user. For each image, check if it
138			* actually present in filesystem (and there is no .nomedia for it) and removes
139			* it from database if it is not present.
140			*
141			* @param string $userId ID of the user for which to remove stale images for
142			* @param int $model Used model
143			* @return \Generator\|int Returns generator during yielding and finally returns int,
144			* which represent number of stale images removed
145			*/
146			private function staleImagesRemovalForUser(string $userId, int $model) {
147
148			$this->fileService->setupFS($userId);
149
150			$this->logDebug(sprintf('Getting all images for user %s', $userId));
151			$allImages = $this->imageMapper->findImages($userId, $model);
152			$this->logDebug(sprintf('Found %d images for user %s', count($allImages), $userId));
153			yield;
154
155			// Find if we stopped somewhere abruptly before. If we are, we need to start from that point.
156			// If there is value, we start from beggining. Important is that:
157			// * There needs to be some (any!) ordering here, we used "id" for ordering key
158			// * New images will be processed, or some might be checked more than once, and that is OK
159			// Important part is that we make continuous progess.
160			$lastChecked = intval($this->config->getUserValue(
161			$userId, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_LAST_CHECKED_KEY, '0'));
162			$this->logDebug(sprintf('Last checked image id for user %s is %d', $userId, $lastChecked));
163			yield;
164
165			// Now filter by those above last checked and sort remaining images
166			$allImages = array_filter($allImages, function ($i) use($lastChecked) {
167			return $i->id > $lastChecked;
168			});
169			usort($allImages, function ($i1, $i2) {
170			return $i1->id <=> $i2->id;
171			});
172			$this->logDebug(sprintf(
173			'After filtering and sorting, there is %d remaining stale images to check for user %s',
174			count($allImages), $userId));
175			yield;
176
177			$handleSharedFiles = $this->config->getAppValue('facerecognition', 'handle-shared-files', 'false');
178
179			// Now iterate and check remaining images
180			$processed = 0;
181			$imagesRemoved = 0;
182			foreach ($allImages as $image) {
183			// Try to get the file to ensure that exist.
184			try {
185			$file = $this->fileService->getFileById($image->getFile(), $userId);
186			} catch (\OCP\Files\NotFoundException $e) {
187			$file = null;
188			}
189
190			// Delete image doesn't exist anymore in filesystem or it is under .nomedia
191			if (($file === null) \|\|
192			($this->fileService->isUnderNoMedia($file)) \|\|
193			($this->fileService->isSharedFile($file) && $handleSharedFiles !== 'true')) {
194			$this->deleteImage($image, $userId);
195			$imagesRemoved++;
196			}
197
198			// Remember last processed image
199			$this->config->setUserValue(
200			$userId, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_LAST_CHECKED_KEY, $image->id);
201
202			// Yield from time to time
203			$processed++;
204			if ($processed % 10 === 0) {
205			$this->logDebug(sprintf('Processed %d/%d stale images for user %s', $processed, count($allImages), $userId));
206			yield;
207			}
208			}
209
210			// Remove this value when we are done, so next cleanup can start from 0
211			$this->config->deleteUserValue($userId, 'facerecognition', StaleImagesRemovalTask::STALE_IMAGES_LAST_CHECKED_KEY);
212			return $imagesRemoved;
213			}
214
215			private function deleteImage(Image $image, string $userId) {
216			$this->logInfo(sprintf('Removing stale image %d for user %s', $image->id, $userId));
217			// note that invalidatePersons depends on existence of faces for a given image,
218			// and we must invalidate before we delete faces!
219			// TODO: this is same method as in Watcher, find where to unify them.
220			$this->personMapper->invalidatePersons($image->id);
221			$this->faceMapper->removeFaces($image->id);
222			$this->imageMapper->delete($image);
223			}
224			}
225

matiasdelellis / facerecognition

Push — shared-storage-experiments ( 606554...2addb2 )

staleImagesRemovalForUser() B

Complexity

Size

Duplication

Code Coverage

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like