Passed
Push — fix-for-non-existing-image ( 3ceadb )
by Branko
03:59
created

ImageProcessingTask::execute()   B

Complexity

Conditions 6
Paths 32

Size

Total Lines 37
Code Lines 25

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 0
CRAP Score 42

Importance

Changes 0
Metric Value
cc 6
eloc 25
nc 32
nop 1
dl 0
loc 37
ccs 0
cts 25
cp 0
crap 42
rs 8.8977
c 0
b 0
f 0
1
<?php
2
/**
3
 * @copyright Copyright (c) 2017, Matias De lellis <[email protected]>
4
 * @copyright Copyright (c) 2018, Branko Kokanovic <[email protected]>
5
 *
6
 * @author Branko Kokanovic <[email protected]>
7
 *
8
 * @license GNU AGPL version 3 or any later version
9
 *
10
 * This program is free software: you can redistribute it and/or modify
11
 * it under the terms of the GNU Affero General Public License as
12
 * published by the Free Software Foundation, either version 3 of the
13
 * License, or (at your option) any later version.
14
 *
15
 * This program is distributed in the hope that it will be useful,
16
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18
 * GNU Affero General Public License for more details.
19
 *
20
 * You should have received a copy of the GNU Affero General Public License
21
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
22
 *
23
 */
24
namespace OCA\FaceRecognition\BackgroundJob\Tasks;
25
26
use OCP\Image as OCP_Image;
27
28
use OCP\Files\File;
29
use OCP\Files\Folder;
30
use OCP\IConfig;
31
use OCP\ITempManager;
32
use OCP\IUser;
33
34
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask;
35
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext;
36
use OCA\FaceRecognition\Db\Face;
37
use OCA\FaceRecognition\Db\Image;
38
use OCA\FaceRecognition\Db\ImageMapper;
39
use OCA\FaceRecognition\Helper\Requirements;
40
use OCA\FaceRecognition\Migration\AddDefaultFaceModel;
41
42
/**
43
 * Plain old PHP object holding all information
44
 * that are needed to process all faces from one image
45
 */
46
class ImageProcessingContext {
47
	/** @var string Path to the image being processed */
48
	private $imagePath;
49
50
	/** @var string Path to temporary, resized image */
51
	private $tempPath;
52
53
	/** @var float Ratio of resized image, when scaling it */
54
	private $ratio;
55
56
	/** @var array<Face> All found faces in image */
57
	private $faces;
58
59
	/**
60
	 * @var bool True if detection should be skipped, but image should be marked as processed.
61
	 * If this is set, $tempPath and $ratio will be invalid and $faces should be empty array.
62
	 */
63
	private $skipDetection;
64
65
	public function __construct(string $imagePath, string $tempPath, float $ratio, bool $skipDetection) {
66
		$this->imagePath = $imagePath;
67
		$this->tempPath = $tempPath;
68
		$this->ratio = $ratio;
69
		$this->faces = array();
70
		$this->skipDetection = $skipDetection;
71
	}
72
73
	public function getImagePath(): string {
74
		return $this->imagePath;
75
	}
76
77
	public function getTempPath(): string {
78
		return $this->tempPath;
79
	}
80
81
	public function getRatio(): float {
82
		return $this->ratio;
83
	}
84
85
	public function getSkipDetection(): bool {
86
		return $this->skipDetection;
87
	}
88
89
	/**
90
	 * Gets all faces
91
	 *
92
	 * @return Face[] Array of faces
93
	 */
94
	public function getFaces(): array {
95
		return $this->faces;
96
	}
97
98
	/**
99
	 * @param array<Face> $faces Array of faces to set
100
	 */
101
	public function setFaces($faces) {
102
		$this->faces = $faces;
103
	}
104
}
105
106
/**
107
 * Taks that get all images that are still not processed and processes them.
108
 * Processing image means that each image is prepared, faces extracted form it,
109
 * and for each found face - face descriptor is extracted.
110
 */
111
class ImageProcessingTask extends FaceRecognitionBackgroundTask {
112
	/** @var IConfig Config */
113
	private $config;
114
115
	/** @var ImageMapper Image mapper*/
116
	protected $imageMapper;
117
118
	/** @var ITempManager */
119
	private $tempManager;
120
121
	/**
122
	 * @param ImageMapper $imageMapper Image mapper
123
	 */
124 1
	public function __construct(IConfig $config, ImageMapper $imageMapper, ITempManager $tempManager) {
125 1
		parent::__construct();
126 1
		$this->config = $config;
127 1
		$this->imageMapper = $imageMapper;
128 1
		$this->tempManager = $tempManager;
129 1
	}
130
131
	/**
132
	 * @inheritdoc
133
	 */
134
	public function description() {
135
		return "Process all images to extract faces";
136
	}
137
138
	/**
139
	 * @inheritdoc
140
	 */
141
	public function execute(FaceRecognitionContext $context) {
142
		$this->setContext($context);
143
144
		$model = intval($this->config->getAppValue('facerecognition', 'model', AddDefaultFaceModel::DEFAULT_FACE_MODEL_ID));
145
		$requirements = new Requirements($context->appManager, $model);
146
147
		$dataDir = rtrim($context->config->getSystemValue('datadirectory', \OC::$SERVERROOT.'/data'), '/');
148
		$images = $context->propertyBag['images'];
149
150
		$cfd = new \CnnFaceDetection($requirements->getFaceDetectionModel());
0 ignored issues
show
Bug introduced by
The type CnnFaceDetection was not found. Maybe you did not declare it correctly or list all dependencies?

The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. excluded_paths: ["lib/*"], you can move it to the dependency path list as follows:

filter:
    dependency_paths: ["lib/*"]

For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths

Loading history...
151
		$fld = new \FaceLandmarkDetection($requirements->getLandmarksDetectionModel());
0 ignored issues
show
Bug introduced by
The type FaceLandmarkDetection was not found. Maybe you did not declare it correctly or list all dependencies?

The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. excluded_paths: ["lib/*"], you can move it to the dependency path list as follows:

filter:
    dependency_paths: ["lib/*"]

For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths

Loading history...
152
		$fr = new \FaceRecognition($requirements->getFaceRecognitionModel());
0 ignored issues
show
Bug introduced by
The type FaceRecognition was not found. Maybe you did not declare it correctly or list all dependencies?

The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. excluded_paths: ["lib/*"], you can move it to the dependency path list as follows:

filter:
    dependency_paths: ["lib/*"]

For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths

Loading history...
153
154
		foreach($images as $image) {
155
			yield;
156
157
			$imageProcessingContext = null;
0 ignored issues
show
Unused Code introduced by
The assignment to $imageProcessingContext is dead and can be removed.
Loading history...
158
			$startMillis = round(microtime(true) * 1000);
159
			try {
160
				$imageProcessingContext = $this->findFaces($cfd, $dataDir, $image);
161
				if (($imageProcessingContext !== null) && ($imageProcessingContext->getSkipDetection() === false)) {
162
					$this->populateDescriptors($fld, $fr, $imageProcessingContext);
163
				}
164
165
				if ($imageProcessingContext !== null) {
166
					$endMillis = round(microtime(true) * 1000);
167
					$duration = max($endMillis - $startMillis, 0);
168
					$this->imageMapper->imageProcessed($image, $imageProcessingContext->getFaces(), $duration);
169
				}
170
			} catch (\Exception $e) {
171
				$this->imageMapper->imageProcessed($image, array(), 0, $e);
172
			} finally {
173
				$this->tempManager->clean();
174
			}
175
		}
176
177
		return true;
178
	}
179
180
	/**
181
	 * Given an image, it finds all faces on it.
182
	 * If image should be skipped, returns null.
183
	 * If there is any error, throws exception
184
	 *
185
	 * @param \CnnFaceDetection $cfd Face detection model
186
	 * @param string $dataDir Directory where data is stored
187
	 * @param Image $image Image to find faces on
188
	 * @return ImageProcessingContext|null Generated context that hold all information needed later for this image
189
	 */
190
	private function findFaces(\CnnFaceDetection $cfd, string $dataDir, Image $image) {
191
		// todo: check if this hits I/O (database, disk...), consider having lazy caching to return user folder from user
192
		$userFolder = $this->context->rootFolder->getUserFolder($image->user);
193
		$userRoot = $userFolder->getParent();
194
		$file = $userRoot->getById($image->file);
195
		if (empty($file)) {
196
			$this->logInfo('File with ID ' . $image->file . ' doesn\'t exist anymore, skipping it');
197
			return null;
198
		}
199
200
		// todo: this concat is wrong with shared files.
201
		$imagePath = $dataDir . $file[0]->getPath();
202
		$this->logInfo('Processing image ' . $imagePath);
203
		$imageProcessingContext = $this->prepareImage($imagePath);
204
		if ($imageProcessingContext->getSkipDetection() === true) {
205
			$this->logInfo('Faces found: 0 (image will be skipped because it is too small)');
206
			return $imageProcessingContext;
207
		}
208
209
		// Detect faces from model
210
		$facesFound = $cfd->detect($imageProcessingContext->getTempPath());
211
212
		// Convert from dictionary of faces to our Face Db Entity
213
		$faces = array();
214
		foreach ($facesFound as $faceFound) {
215
			$face = Face::fromModel($image->getId(), $faceFound);
216
			$face->normalizeSize($imageProcessingContext->getRatio());
217
			$faces[] = $face;
218
		}
219
220
		$imageProcessingContext->setFaces($faces);
221
		$this->logInfo('Faces found: ' . count($faces));
222
223
		return $imageProcessingContext;
224
	}
225
226
	/**
227
	 * Given an image, it will rotate, scale and save image to temp location, ready to be consumed by pdlib.
228
	 *
229
	 * @param string $imagePath Path to image on disk
230
	 *
231
	 * @return ImageProcessingContext Generated context that hold all information needed later for this image.
232
	 */
233
	private function prepareImage(string $imagePath) {
234
		$image = new OCP_Image(null, $this->context->logger->getLogger(), $this->context->config);
235
		$image->loadFromFile($imagePath);
236
		$image->fixOrientation();
237
		if (!$image->valid()) {
238
			throw new \RuntimeException("Image is not valid, probably cannot be loaded");
239
		}
240
241
		// Ignore processing of images that are not large enough.
242
		$minImageSize = intval($this->config->getAppValue('facerecognition', 'min_image_size', '512'));
243
		if ((imagesx($image->resource()) < $minImageSize) || (imagesy($image->resource()) < $minImageSize)) {
244
			return new ImageProcessingContext($imagePath, "", -1, true);
245
		}
246
247
		// Based on amount on memory PHP have, we will determine maximum amount of image size that we need to scale to.
248
		// This reasoning and calculations are all based on analysis given here:
249
		// https://github.com/matiasdelellis/facerecognition/wiki/Performance-analysis-of-DLib%E2%80%99s-CNN-face-detection
250
		$allowedMemory = $this->context->propertyBag['memory'];
251
		$maxImageArea = intval((0.75 * $allowedMemory) / 1024); // in pixels^2
252
		$ratio = $this->resizeImage($image, $maxImageArea);
253
254
		$tempfile = $this->tempManager->getTemporaryFile(pathinfo($imagePath, PATHINFO_EXTENSION));
255
		$image->save($tempfile);
256
		return new ImageProcessingContext($imagePath, $tempfile, $ratio, false);
257
	}
258
259
	/**
260
	 * Resizes the image to reach max image area, but preserving ratio.
261
	 * Stolen and adopted from OC_Image->resize() (difference is that this returns ratio of resize.)
262
	 *
263
	 * @param OC_Image $image Image to resize
0 ignored issues
show
Bug introduced by
The type OCA\FaceRecognition\BackgroundJob\Tasks\OC_Image was not found. Maybe you did not declare it correctly or list all dependencies?

The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. excluded_paths: ["lib/*"], you can move it to the dependency path list as follows:

filter:
    dependency_paths: ["lib/*"]

For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths

Loading history...
264
	 * @param int $maxImageArea The maximum size of image we can handle (in pixels^2).
265
	 *
266
	 * @return float Ratio of resize. 1 if there was no resize
267
	 */
268 1
	public function resizeImage(OCP_Image $image, int $maxImageArea): float {
269 1
		if (!$image->valid()) {
270
			$message = "Image is not valid, probably cannot be loaded";
271
			$this->logInfo($message);
272
			throw new \RuntimeException($message);
273
		}
274
275 1
		$widthOrig = imagesx($image->resource());
276 1
		$heightOrig = imagesy($image->resource());
277 1
		if (($widthOrig <= 0) || ($heightOrig <= 0)) {
278
			$message = "Image is having non-positive width or height, cannot continue";
279
			$this->logInfo($message);
280
			throw new \RuntimeException($message);
281
		}
282
283 1
		$areaRatio = $maxImageArea / ($widthOrig * $heightOrig);
284 1
		$scaleFactor = sqrt($areaRatio);
285
286 1
		$newWidth = intval(round($widthOrig * $scaleFactor));
287 1
		$newHeight = intval(round($heightOrig * $scaleFactor));
288
289 1
		$success = $image->preciseResize($newWidth, $newHeight);
290 1
		if ($success === false) {
291
			throw new \RuntimeException("Error during image resize");
292
		}
293
294 1
		$this->logDebug(sprintf('Image scaled from %dx%d to %dx%d (since max image area is %d pixels^2)',
295 1
			$newWidth, $newHeight, $widthOrig, $heightOrig, $maxImageArea));
296
297 1
		return 1 / $scaleFactor;
298
	}
299
300
	/**
301
	 * Gets all face descriptors in a given image processing context. Populates "descriptor" in array of faces.
302
	 *
303
	 * @param \FaceLandmarkDetection $fld Landmark detection model
304
	 * @param \FaceRecognition $fr Face recognition model
305
	 * @param ImageProcessingContext Image processing context
306
	 */
307
	private function populateDescriptors(\FaceLandmarkDetection $fld, \FaceRecognition $fr, ImageProcessingContext $imageProcessingContext) {
308
		$faces = $imageProcessingContext->getFaces();
309
310
		foreach($faces as &$face) {
311
			$tempfilePath = $this->cropFace($imageProcessingContext->getImagePath(), $face);
312
313
			// Usually, second argument to detect should be just $face. However, since we are doing image acrobatics
314
			// and already have cropped image, bounding box for landmark detection is now complete (cropped) image!
315
			$landmarks = $fld->detect($tempfilePath, array(
316
				"left" => 0, "top" => 0, "bottom" => $face->height(), "right" => $face->width()));
317
			$descriptor = $fr->computeDescriptor($tempfilePath, $landmarks);
318
			$face->descriptor = $descriptor;
319
		}
320
	}
321
322
	private function cropFace(string $imagePath, Face $face): string {
323
		// todo: we are loading same image two times, fix this
324
		$image = new OCP_Image(null, $this->context->logger->getLogger(), $this->context->config);
325
		$image->loadFromFile($imagePath);
326
		$image->fixOrientation();
327
		$success = $image->crop($face->left, $face->top, $face->width(), $face->height());
328
		if ($success === false) {
329
			throw new \RuntimeException("Error during image cropping");
330
		}
331
332
		$tempfile = $this->tempManager->getTemporaryFile(pathinfo($imagePath, PATHINFO_EXTENSION));
333
		$image->save($tempfile);
334
		return $tempfile;
335
	}
336
}