Passed
Pull Request — master (#232)
by Matias
05:40 queued 04:10
created

ImageProcessingTask::calculateMaxImageArea()   A

Complexity

Conditions 4
Paths 3

Size

Total Lines 27
Code Lines 10

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 6
CRAP Score 5.024

Importance

Changes 0
Metric Value
cc 4
eloc 10
nc 3
nop 0
dl 0
loc 27
ccs 6
cts 10
cp 0.6
crap 5.024
rs 9.9332
c 0
b 0
f 0
1
<?php
2
/**
3
 * @copyright Copyright (c) 2017-2020 Matias De lellis <[email protected]>
4
 * @copyright Copyright (c) 2018, Branko Kokanovic <[email protected]>
5
 *
6
 * @author Branko Kokanovic <[email protected]>
7
 *
8
 * @license GNU AGPL version 3 or any later version
9
 *
10
 * This program is free software: you can redistribute it and/or modify
11
 * it under the terms of the GNU Affero General Public License as
12
 * published by the Free Software Foundation, either version 3 of the
13
 * License, or (at your option) any later version.
14
 *
15
 * This program is distributed in the hope that it will be useful,
16
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18
 * GNU Affero General Public License for more details.
19
 *
20
 * You should have received a copy of the GNU Affero General Public License
21
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
22
 *
23
 */
24
namespace OCA\FaceRecognition\BackgroundJob\Tasks;
25
26
use OCP\Image as OCP_Image;
27
28
use OCP\Files\File;
29
use OCP\Files\Folder;
30
use OCP\IUser;
31
32
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask;
33
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext;
34
35
use OCA\FaceRecognition\Db\Face;
36
use OCA\FaceRecognition\Db\Image;
37
use OCA\FaceRecognition\Db\ImageMapper;
38
39
use OCA\FaceRecognition\Model\IModel;
40
use OCA\FaceRecognition\Model\ModelManager;
41
42
use OCA\FaceRecognition\Service\FileService;
43
use OCA\FaceRecognition\Service\SettingsService;
44
45
/**
46
 * Plain old PHP object holding all information
47
 * that are needed to process all faces from one image
48
 */
49
class ImageProcessingContext {
50
	/** @var string Path to the image being processed */
51
	private $imagePath;
52
53
	/** @var string Path to temporary, resized image */
54
	private $tempPath;
55
56
	/** @var float Ratio of resized image, when scaling it */
57
	private $ratio;
58
59
	/** @var array<Face> All found faces in image */
60
	private $faces;
61
62
	/**
63
	 * @var bool True if detection should be skipped, but image should be marked as processed.
64
	 * If this is set, $tempPath and $ratio will be invalid and $faces should be empty array.
65
	 */
66
	private $skipDetection;
67
68 3
	public function __construct(string $imagePath, string $tempPath, float $ratio, bool $skipDetection) {
69 3
		$this->imagePath = $imagePath;
70 3
		$this->tempPath = $tempPath;
71 3
		$this->ratio = $ratio;
72 3
		$this->faces = array();
73 3
		$this->skipDetection = $skipDetection;
74 3
	}
75
76
	public function getImagePath(): string {
77
		return $this->imagePath;
78
	}
79
80 2
	public function getTempPath(): string {
81 2
		return $this->tempPath;
82
	}
83
84 1
	public function getRatio(): float {
85 1
		return $this->ratio;
86
	}
87
88 3
	public function getSkipDetection(): bool {
89 3
		return $this->skipDetection;
90
	}
91
92
	/**
93
	 * Gets all faces
94
	 *
95
	 * @return Face[] Array of faces
96
	 */
97 3
	public function getFaces(): array {
98 3
		return $this->faces;
99
	}
100
101
	/**
102
	 * @param array<Face> $faces Array of faces to set
103
	 */
104 2
	public function setFaces($faces) {
105 2
		$this->faces = $faces;
106 2
	}
107
}
108
109
/**
110
 * Taks that get all images that are still not processed and processes them.
111
 * Processing image means that each image is prepared, faces extracted form it,
112
 * and for each found face - face descriptor is extracted.
113
 */
114
class ImageProcessingTask extends FaceRecognitionBackgroundTask {
115
	/** @var ImageMapper Image mapper*/
116
	protected $imageMapper;
117
118
	/** @var FileService */
119
	protected $fileService;
120
121
	/** @var SettingsService */
122
	protected $settingsService;
123
124
	/** @var ModelManager */
125
	protected $modelManager;
126
127
	/** @var IModel */
128
	private $model;
129
130
	/** @var int|null Maximum image area (cached, so it is not recalculated for each image) */
131
	private $maxImageAreaCached;
132
133
	/**
134
	 * @param ImageMapper $imageMapper Image mapper
135
	 * @param FileService $fileService
136
	 * @param SettingsService $settingsService
137
	 * @param ModelManager $modelManager Model manager
138
	 */
139 5
	public function __construct(ImageMapper     $imageMapper,
140
	                            FileService     $fileService,
141
	                            SettingsService $settingsService,
142
	                            ModelManager    $modelManager)
143
	{
144 5
		parent::__construct();
145
146 5
		$this->imageMapper        = $imageMapper;
147 5
		$this->fileService        = $fileService;
148 5
		$this->settingsService    = $settingsService;
149 5
		$this->modelManager       = $modelManager;
150
151 5
		$this->model              = null;
152 5
		$this->maxImageAreaCached = null;
153 5
	}
154
155
	/**
156
	 * @inheritdoc
157
	 */
158 4
	public function description() {
159 4
		return "Process all images to extract faces";
160
	}
161
162
	/**
163
	 * @inheritdoc
164
	 */
165 4
	public function execute(FaceRecognitionContext $context) {
166 4
		$this->setContext($context);
167
168 4
		$this->logInfo('NOTE: Starting face recognition. If you experience random crashes after this point, please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ');
169
170
		// Get current model.
171 4
		$this->model = $this->modelManager->getCurrentModel();
172
173
		// Open model.
174 4
		$this->model->open();
175
176 4
		$images = $context->propertyBag['images'];
177 4
		foreach($images as $image) {
178 4
			yield;
179
180 4
			$startMillis = round(microtime(true) * 1000);
181
182
			try {
183 4
				$imageProcessingContext = $this->findFaces($this->model, $image);
184
185 3
				if (($imageProcessingContext !== null) && ($imageProcessingContext->getSkipDetection() === false)) {
186 2
					$this->populateDescriptors($this->model, $imageProcessingContext);
187
				}
188
189 3
				if ($imageProcessingContext === null) {
190
					continue;
191
				}
192
193 3
				$endMillis = round(microtime(true) * 1000);
194 3
				$duration = max($endMillis - $startMillis, 0);
195 3
				$this->imageMapper->imageProcessed($image, $imageProcessingContext->getFaces(), $duration);
196 1
			} catch (\Exception $e) {
197 1
				if ($e->getMessage() === "std::bad_alloc") {
198
					throw new \RuntimeException("Not enough memory to run face recognition! Please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ");
199
				}
200 1
				$this->logInfo('Faces found: 0. Image will be skipped because of the following error: ' . $e->getMessage());
201 1
				$this->logDebug($e);
202 1
				$this->imageMapper->imageProcessed($image, array(), 0, $e);
203 4
			} finally {
204 4
				$this->fileService->clean();
205
			}
206
		}
207
208 4
		return true;
209
	}
210
211
	/**
212
	 * Given an image, it finds all faces on it.
213
	 * If image should be skipped, returns null.
214
	 * If there is any error, throws exception
215
	 *
216
	 * @param IModel $model Resnet model
217
	 * @param Image $image Image to find faces on
218
	 * @return ImageProcessingContext|null Generated context that hold all information needed later for this image
219
	 */
220 4
	private function findFaces(IModel $model, Image $image) {
221
		// todo: check if this hits I/O (database, disk...), consider having lazy caching to return user folder from user
222 4
		$file = $this->fileService->getFileById($image->getFile(), $image->getUser());
223
224 4
		if (empty($file)) {
225
			// If we cannot find a file probably it was deleted out of our control and we must clean our tables.
226
			$this->settingsService->setNeedRemoveStaleImages(true, $image->user);
227
			$this->logInfo('File with ID ' . $image->file . ' doesn\'t exist anymore, skipping it');
228
			return null;
229
		}
230
231 4
		$imagePath = $this->fileService->getLocalFile($file);
232
233 4
		$this->logInfo('Processing image ' . $imagePath);
234 4
		$imageProcessingContext = $this->prepareImage($imagePath);
235 3
		if ($imageProcessingContext->getSkipDetection() === true) {
236 1
			$this->logInfo('Faces found: 0 (image will be skipped because it is too small)');
237 1
			return $imageProcessingContext;
238
		}
239
240
		// Detect faces from model
241 2
		$facesFound = $model->detectFaces($imageProcessingContext->getTempPath());
242
243
		// Convert from dictionary of faces to our Face Db Entity
244 2
		$faces = array();
245 2
		foreach ($facesFound as $faceFound) {
246 1
			$face = Face::fromModel($image->getId(), $faceFound);
247 1
			$face->normalizeSize($imageProcessingContext->getRatio());
248 1
			$faces[] = $face;
249
		}
250
251 2
		$imageProcessingContext->setFaces($faces);
252 2
		$this->logInfo('Faces found: ' . count($faces));
253
254 2
		return $imageProcessingContext;
255
	}
256
257
	/**
258
	 * Given an image, it will rotate, scale and save image to temp location, ready to be consumed by pdlib.
259
	 *
260
	 * @param string $imagePath Path to image on disk
261
	 *
262
	 * @return ImageProcessingContext Generated context that hold all information needed later for this image.
263
	 */
264 4
	private function prepareImage(string $imagePath) {
265 4
		$image = new OCP_Image(null, $this->context->logger->getLogger(), $this->context->config);
266 4
		$image->loadFromFile($imagePath);
267 3
		$image->fixOrientation();
268
269 3
		if (!$image->valid()) {
270
			throw new \RuntimeException("Image is not valid, probably cannot be loaded");
271
		}
272
273
		// Ignore processing of images that are not large enough.
274 3
		$minImageSize = $this->settingsService->getMinimumImageSize();
275 3
		if ((imagesx($image->resource()) < $minImageSize) || (imagesy($image->resource()) < $minImageSize)) {
276 1
			return new ImageProcessingContext($imagePath, "", -1, true);
277
		}
278
279 2
		$maxImageArea = $this->getMaxImageArea();
280 2
		$ratio = $this->resizeImage($image, $maxImageArea);
281
282 2
		$tempfile = $this->fileService->getTemporaryFile(pathinfo($imagePath, PATHINFO_EXTENSION));
283 2
		$image->save($tempfile);
284
285 2
		return new ImageProcessingContext($imagePath, $tempfile, $ratio, false);
286
	}
287
288
	/**
289
	 * Resizes the image to reach max image area, but preserving ratio.
290
	 * Stolen and adopted from OC_Image->resize() (difference is that this returns ratio of resize.)
291
	 *
292
	 * @param Image $image Image to resize
293
	 * @param int $maxImageArea The maximum size of image we can handle (in pixels^2).
294
	 *
295
	 * @return float Ratio of resize. 1 if there was no resize
296
	 */
297 3
	public function resizeImage(OCP_Image $image, int $maxImageArea): float {
298 3
		if (!$image->valid()) {
299
			$message = "Image is not valid, probably cannot be loaded";
300
			$this->logInfo($message);
301
			throw new \RuntimeException($message);
302
		}
303
304 3
		$widthOrig = imagesx($image->resource());
305 3
		$heightOrig = imagesy($image->resource());
306 3
		if (($widthOrig <= 0) || ($heightOrig <= 0)) {
307
			$message = "Image is having non-positive width or height, cannot continue";
308
			$this->logInfo($message);
309
			throw new \RuntimeException($message);
310
		}
311
312 3
		$areaRatio = $maxImageArea / ($widthOrig * $heightOrig);
313 3
		$scaleFactor = sqrt($areaRatio);
314
315 3
		$newWidth = intval(round($widthOrig * $scaleFactor));
316 3
		$newHeight = intval(round($heightOrig * $scaleFactor));
317
318 3
		$success = $image->preciseResize($newWidth, $newHeight);
319 3
		if ($success === false) {
320
			throw new \RuntimeException("Error during image resize");
321
		}
322
323 3
		$this->logDebug(sprintf('Image scaled from %dx%d to %dx%d (since max image area is %d pixels^2)',
324 3
			$widthOrig, $heightOrig, $newWidth, $newHeight, $maxImageArea));
325
326 3
		return 1 / $scaleFactor;
327
	}
328
329
	/**
330
	 * Gets all face descriptors in a given image processing context. Populates "descriptor" in array of faces.
331
	 *
332
	 * @param IModel $model Resnet model
333
	 * @param ImageProcessingContext Image processing context
334
	 */
335 2
	private function populateDescriptors(IModel $model, ImageProcessingContext $imageProcessingContext) {
336 2
		$faces = $imageProcessingContext->getFaces();
337
338 2
		foreach($faces as &$face) {
339
			// For each face, we want to detect landmarks and compute descriptors.
340
			// We use already resized image (from temp, used to detect faces) for this.
341
			// (better would be to work with original image, but that will require
342
			// another orientation fix and another save to the temp)
343
			// But, since our face coordinates are already changed to align to original image,
344
			// we need to fix them up to align them to temp image here.
345 1
			$normalizedFace = clone $face;
346 1
			$normalizedFace->normalizeSize(1.0 / $imageProcessingContext->getRatio());
347
348
			// We are getting face landmarks from already prepared (temp) image (resized and with orienation fixed).
349 1
			$landmarks = $model->detectLandmarks($imageProcessingContext->getTempPath(), array(
350 1
				"left" => $normalizedFace->left, "top" => $normalizedFace->top,
351 1
				"bottom" => $normalizedFace->bottom, "right" => $normalizedFace->right));
352 1
			$face->landmarks = $landmarks['parts'];
353
354 1
			$descriptor = $model->computeDescriptor($imageProcessingContext->getTempPath(), $landmarks);
355 1
			$face->descriptor = $descriptor;
356
		}
357 2
	}
358
359
	/**
360
	 * Obtains max image area lazily (from cache, or calculates it and puts it to cache)
361
	 *
362
	 * @return int Max image area (in pixels^2)
363
	 */
364 2
	private function getMaxImageArea(): int {
365
		// First check if is cached
366
		//
367 2
		if (!is_null($this->maxImageAreaCached)) {
368
			return $this->maxImageAreaCached;
369
		}
370
371
		// Get this setting on main app_config.
372
		// Note that this option has lower and upper limits and validations
373 2
		$this->maxImageAreaCached = $this->settingsService->getAnalysisImageArea();
374
375
		// Check if admin override it in config and it is valid value
376
		//
377 2
		$maxImageArea = $this->settingsService->getMaximumImageArea();
378 2
		if ($maxImageArea > 0) {
379 2
			$this->maxImageAreaCached = $maxImageArea;
380
		}
381
		// Also check if we are provided value from command line.
382
		//
383 2
		if ((array_key_exists('max_image_area', $this->context->propertyBag)) &&
384 2
		    (!is_null($this->context->propertyBag['max_image_area']))) {
385
			$this->maxImageAreaCached = $this->context->propertyBag['max_image_area'];
386
		}
387
388 2
		return $this->maxImageAreaCached;
389
	}
390
391
}