1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* @copyright Copyright (c) 2017-2020 Matias De lellis <[email protected]> |
4
|
|
|
* @copyright Copyright (c) 2018, Branko Kokanovic <[email protected]> |
5
|
|
|
* |
6
|
|
|
* @author Branko Kokanovic <[email protected]> |
7
|
|
|
* |
8
|
|
|
* @license GNU AGPL version 3 or any later version |
9
|
|
|
* |
10
|
|
|
* This program is free software: you can redistribute it and/or modify |
11
|
|
|
* it under the terms of the GNU Affero General Public License as |
12
|
|
|
* published by the Free Software Foundation, either version 3 of the |
13
|
|
|
* License, or (at your option) any later version. |
14
|
|
|
* |
15
|
|
|
* This program is distributed in the hope that it will be useful, |
16
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
17
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
18
|
|
|
* GNU Affero General Public License for more details. |
19
|
|
|
* |
20
|
|
|
* You should have received a copy of the GNU Affero General Public License |
21
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
22
|
|
|
* |
23
|
|
|
*/ |
24
|
|
|
namespace OCA\FaceRecognition\BackgroundJob\Tasks; |
25
|
|
|
|
26
|
|
|
use OCP\Image as OCP_Image; |
27
|
|
|
|
28
|
|
|
use OCP\Files\File; |
29
|
|
|
use OCP\Files\Folder; |
30
|
|
|
use OCP\IUser; |
31
|
|
|
|
32
|
|
|
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask; |
33
|
|
|
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext; |
34
|
|
|
|
35
|
|
|
use OCA\FaceRecognition\Db\Face; |
36
|
|
|
use OCA\FaceRecognition\Db\Image; |
37
|
|
|
use OCA\FaceRecognition\Db\ImageMapper; |
38
|
|
|
|
39
|
|
|
use OCA\FaceRecognition\Model\IModel; |
40
|
|
|
use OCA\FaceRecognition\Model\ModelManager; |
41
|
|
|
|
42
|
|
|
use OCA\FaceRecognition\Service\FileService; |
43
|
|
|
use OCA\FaceRecognition\Service\SettingsService; |
44
|
|
|
|
45
|
|
|
/** |
46
|
|
|
* Plain old PHP object holding all information |
47
|
|
|
* that are needed to process all faces from one image |
48
|
|
|
*/ |
49
|
|
|
class ImageProcessingContext { |
50
|
|
|
/** @var string Path to the image being processed */ |
51
|
|
|
private $imagePath; |
52
|
|
|
|
53
|
|
|
/** @var string Path to temporary, resized image */ |
54
|
|
|
private $tempPath; |
55
|
|
|
|
56
|
|
|
/** @var float Ratio of resized image, when scaling it */ |
57
|
|
|
private $ratio; |
58
|
|
|
|
59
|
|
|
/** @var array<Face> All found faces in image */ |
60
|
|
|
private $faces; |
61
|
|
|
|
62
|
|
|
/** |
63
|
|
|
* @var bool True if detection should be skipped, but image should be marked as processed. |
64
|
|
|
* If this is set, $tempPath and $ratio will be invalid and $faces should be empty array. |
65
|
|
|
*/ |
66
|
|
|
private $skipDetection; |
67
|
|
|
|
68
|
3 |
|
public function __construct(string $imagePath, string $tempPath, float $ratio, bool $skipDetection) { |
69
|
3 |
|
$this->imagePath = $imagePath; |
70
|
3 |
|
$this->tempPath = $tempPath; |
71
|
3 |
|
$this->ratio = $ratio; |
72
|
3 |
|
$this->faces = array(); |
73
|
3 |
|
$this->skipDetection = $skipDetection; |
74
|
3 |
|
} |
75
|
|
|
|
76
|
|
|
public function getImagePath(): string { |
77
|
|
|
return $this->imagePath; |
78
|
|
|
} |
79
|
|
|
|
80
|
2 |
|
public function getTempPath(): string { |
81
|
2 |
|
return $this->tempPath; |
82
|
|
|
} |
83
|
|
|
|
84
|
1 |
|
public function getRatio(): float { |
85
|
1 |
|
return $this->ratio; |
86
|
|
|
} |
87
|
|
|
|
88
|
3 |
|
public function getSkipDetection(): bool { |
89
|
3 |
|
return $this->skipDetection; |
90
|
|
|
} |
91
|
|
|
|
92
|
|
|
/** |
93
|
|
|
* Gets all faces |
94
|
|
|
* |
95
|
|
|
* @return Face[] Array of faces |
96
|
|
|
*/ |
97
|
3 |
|
public function getFaces(): array { |
98
|
3 |
|
return $this->faces; |
99
|
|
|
} |
100
|
|
|
|
101
|
|
|
/** |
102
|
|
|
* @param array<Face> $faces Array of faces to set |
103
|
|
|
*/ |
104
|
2 |
|
public function setFaces($faces) { |
105
|
2 |
|
$this->faces = $faces; |
106
|
2 |
|
} |
107
|
|
|
} |
108
|
|
|
|
109
|
|
|
/** |
110
|
|
|
* Taks that get all images that are still not processed and processes them. |
111
|
|
|
* Processing image means that each image is prepared, faces extracted form it, |
112
|
|
|
* and for each found face - face descriptor is extracted. |
113
|
|
|
*/ |
114
|
|
|
class ImageProcessingTask extends FaceRecognitionBackgroundTask { |
115
|
|
|
/** @var ImageMapper Image mapper*/ |
116
|
|
|
protected $imageMapper; |
117
|
|
|
|
118
|
|
|
/** @var FileService */ |
119
|
|
|
protected $fileService; |
120
|
|
|
|
121
|
|
|
/** @var SettingsService */ |
122
|
|
|
protected $settingsService; |
123
|
|
|
|
124
|
|
|
/** @var ModelManager */ |
125
|
|
|
protected $modelManager; |
126
|
|
|
|
127
|
|
|
/** @var IModel */ |
128
|
|
|
private $model; |
129
|
|
|
|
130
|
|
|
/** @var int|null Maximum image area (cached, so it is not recalculated for each image) */ |
131
|
|
|
private $maxImageAreaCached; |
132
|
|
|
|
133
|
|
|
/** |
134
|
|
|
* @param ImageMapper $imageMapper Image mapper |
135
|
|
|
* @param FileService $fileService |
136
|
|
|
* @param SettingsService $settingsService |
137
|
|
|
* @param ModelManager $modelManager Model manager |
138
|
|
|
*/ |
139
|
5 |
|
public function __construct(ImageMapper $imageMapper, |
140
|
|
|
FileService $fileService, |
141
|
|
|
SettingsService $settingsService, |
142
|
|
|
ModelManager $modelManager) |
143
|
|
|
{ |
144
|
5 |
|
parent::__construct(); |
145
|
|
|
|
146
|
5 |
|
$this->imageMapper = $imageMapper; |
147
|
5 |
|
$this->fileService = $fileService; |
148
|
5 |
|
$this->settingsService = $settingsService; |
149
|
5 |
|
$this->modelManager = $modelManager; |
150
|
|
|
|
151
|
5 |
|
$this->model = null; |
152
|
5 |
|
$this->maxImageAreaCached = null; |
153
|
5 |
|
} |
154
|
|
|
|
155
|
|
|
/** |
156
|
|
|
* @inheritdoc |
157
|
|
|
*/ |
158
|
4 |
|
public function description() { |
159
|
4 |
|
return "Process all images to extract faces"; |
160
|
|
|
} |
161
|
|
|
|
162
|
|
|
/** |
163
|
|
|
* @inheritdoc |
164
|
|
|
*/ |
165
|
4 |
|
public function execute(FaceRecognitionContext $context) { |
166
|
4 |
|
$this->setContext($context); |
167
|
|
|
|
168
|
4 |
|
$this->logInfo('NOTE: Starting face recognition. If you experience random crashes after this point, please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ'); |
169
|
|
|
|
170
|
|
|
// Get current model. |
171
|
4 |
|
$this->model = $this->modelManager->getCurrentModel(); |
172
|
|
|
|
173
|
|
|
// Open model. |
174
|
4 |
|
$this->model->open(); |
175
|
|
|
|
176
|
4 |
|
$images = $context->propertyBag['images']; |
177
|
4 |
|
foreach($images as $image) { |
178
|
4 |
|
yield; |
179
|
|
|
|
180
|
4 |
|
$startMillis = round(microtime(true) * 1000); |
181
|
|
|
|
182
|
|
|
try { |
183
|
4 |
|
$imageProcessingContext = $this->findFaces($this->model, $image); |
184
|
|
|
|
185
|
3 |
|
if (($imageProcessingContext !== null) && ($imageProcessingContext->getSkipDetection() === false)) { |
186
|
2 |
|
$this->populateDescriptors($this->model, $imageProcessingContext); |
187
|
|
|
} |
188
|
|
|
|
189
|
3 |
|
if ($imageProcessingContext === null) { |
190
|
|
|
continue; |
191
|
|
|
} |
192
|
|
|
|
193
|
3 |
|
$endMillis = round(microtime(true) * 1000); |
194
|
3 |
|
$duration = max($endMillis - $startMillis, 0); |
195
|
3 |
|
$this->imageMapper->imageProcessed($image, $imageProcessingContext->getFaces(), $duration); |
196
|
1 |
|
} catch (\Exception $e) { |
197
|
1 |
|
if ($e->getMessage() === "std::bad_alloc") { |
198
|
|
|
throw new \RuntimeException("Not enough memory to run face recognition! Please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ"); |
199
|
|
|
} |
200
|
1 |
|
$this->logInfo('Faces found: 0. Image will be skipped because of the following error: ' . $e->getMessage()); |
201
|
1 |
|
$this->logDebug($e); |
202
|
1 |
|
$this->imageMapper->imageProcessed($image, array(), 0, $e); |
203
|
4 |
|
} finally { |
204
|
4 |
|
$this->fileService->clean(); |
205
|
|
|
} |
206
|
|
|
} |
207
|
|
|
|
208
|
4 |
|
return true; |
209
|
|
|
} |
210
|
|
|
|
211
|
|
|
/** |
212
|
|
|
* Given an image, it finds all faces on it. |
213
|
|
|
* If image should be skipped, returns null. |
214
|
|
|
* If there is any error, throws exception |
215
|
|
|
* |
216
|
|
|
* @param IModel $model Resnet model |
217
|
|
|
* @param Image $image Image to find faces on |
218
|
|
|
* @return ImageProcessingContext|null Generated context that hold all information needed later for this image |
219
|
|
|
*/ |
220
|
4 |
|
private function findFaces(IModel $model, Image $image) { |
221
|
|
|
// todo: check if this hits I/O (database, disk...), consider having lazy caching to return user folder from user |
222
|
4 |
|
$file = $this->fileService->getFileById($image->getFile(), $image->getUser()); |
223
|
|
|
|
224
|
4 |
|
if (empty($file)) { |
225
|
|
|
// If we cannot find a file probably it was deleted out of our control and we must clean our tables. |
226
|
|
|
$this->settingsService->setNeedRemoveStaleImages(true, $image->user); |
227
|
|
|
$this->logInfo('File with ID ' . $image->file . ' doesn\'t exist anymore, skipping it'); |
228
|
|
|
return null; |
229
|
|
|
} |
230
|
|
|
|
231
|
4 |
|
$imagePath = $this->fileService->getLocalFile($file); |
232
|
|
|
|
233
|
4 |
|
$this->logInfo('Processing image ' . $imagePath); |
234
|
4 |
|
$imageProcessingContext = $this->prepareImage($imagePath); |
235
|
3 |
|
if ($imageProcessingContext->getSkipDetection() === true) { |
236
|
1 |
|
$this->logInfo('Faces found: 0 (image will be skipped because it is too small)'); |
237
|
1 |
|
return $imageProcessingContext; |
238
|
|
|
} |
239
|
|
|
|
240
|
|
|
// Detect faces from model |
241
|
2 |
|
$facesFound = $model->detectFaces($imageProcessingContext->getTempPath()); |
242
|
|
|
|
243
|
|
|
// Convert from dictionary of faces to our Face Db Entity |
244
|
2 |
|
$faces = array(); |
245
|
2 |
|
foreach ($facesFound as $faceFound) { |
246
|
1 |
|
$face = Face::fromModel($image->getId(), $faceFound); |
247
|
1 |
|
$face->normalizeSize($imageProcessingContext->getRatio()); |
248
|
1 |
|
$faces[] = $face; |
249
|
|
|
} |
250
|
|
|
|
251
|
2 |
|
$imageProcessingContext->setFaces($faces); |
252
|
2 |
|
$this->logInfo('Faces found: ' . count($faces)); |
253
|
|
|
|
254
|
2 |
|
return $imageProcessingContext; |
255
|
|
|
} |
256
|
|
|
|
257
|
|
|
/** |
258
|
|
|
* Given an image, it will rotate, scale and save image to temp location, ready to be consumed by pdlib. |
259
|
|
|
* |
260
|
|
|
* @param string $imagePath Path to image on disk |
261
|
|
|
* |
262
|
|
|
* @return ImageProcessingContext Generated context that hold all information needed later for this image. |
263
|
|
|
*/ |
264
|
4 |
|
private function prepareImage(string $imagePath) { |
265
|
4 |
|
$image = new OCP_Image(null, $this->context->logger->getLogger(), $this->context->config); |
266
|
4 |
|
$image->loadFromFile($imagePath); |
267
|
3 |
|
$image->fixOrientation(); |
268
|
|
|
|
269
|
3 |
|
if (!$image->valid()) { |
270
|
|
|
throw new \RuntimeException("Image is not valid, probably cannot be loaded"); |
271
|
|
|
} |
272
|
|
|
|
273
|
|
|
// Ignore processing of images that are not large enough. |
274
|
3 |
|
$minImageSize = $this->settingsService->getMinimumImageSize(); |
275
|
3 |
|
if ((imagesx($image->resource()) < $minImageSize) || (imagesy($image->resource()) < $minImageSize)) { |
276
|
1 |
|
return new ImageProcessingContext($imagePath, "", -1, true); |
277
|
|
|
} |
278
|
|
|
|
279
|
2 |
|
$maxImageArea = $this->getMaxImageArea(); |
280
|
2 |
|
$ratio = $this->resizeImage($image, $maxImageArea); |
281
|
|
|
|
282
|
2 |
|
$tempfile = $this->fileService->getTemporaryFile(pathinfo($imagePath, PATHINFO_EXTENSION)); |
283
|
2 |
|
$image->save($tempfile); |
284
|
|
|
|
285
|
2 |
|
return new ImageProcessingContext($imagePath, $tempfile, $ratio, false); |
286
|
|
|
} |
287
|
|
|
|
288
|
|
|
/** |
289
|
|
|
* Resizes the image to reach max image area, but preserving ratio. |
290
|
|
|
* Stolen and adopted from OC_Image->resize() (difference is that this returns ratio of resize.) |
291
|
|
|
* |
292
|
|
|
* @param Image $image Image to resize |
293
|
|
|
* @param int $maxImageArea The maximum size of image we can handle (in pixels^2). |
294
|
|
|
* |
295
|
|
|
* @return float Ratio of resize. 1 if there was no resize |
296
|
|
|
*/ |
297
|
3 |
|
public function resizeImage(OCP_Image $image, int $maxImageArea): float { |
298
|
3 |
|
if (!$image->valid()) { |
299
|
|
|
$message = "Image is not valid, probably cannot be loaded"; |
300
|
|
|
$this->logInfo($message); |
301
|
|
|
throw new \RuntimeException($message); |
302
|
|
|
} |
303
|
|
|
|
304
|
3 |
|
$widthOrig = imagesx($image->resource()); |
305
|
3 |
|
$heightOrig = imagesy($image->resource()); |
306
|
3 |
|
if (($widthOrig <= 0) || ($heightOrig <= 0)) { |
307
|
|
|
$message = "Image is having non-positive width or height, cannot continue"; |
308
|
|
|
$this->logInfo($message); |
309
|
|
|
throw new \RuntimeException($message); |
310
|
|
|
} |
311
|
|
|
|
312
|
3 |
|
$areaRatio = $maxImageArea / ($widthOrig * $heightOrig); |
313
|
3 |
|
$scaleFactor = sqrt($areaRatio); |
314
|
|
|
|
315
|
3 |
|
$newWidth = intval(round($widthOrig * $scaleFactor)); |
316
|
3 |
|
$newHeight = intval(round($heightOrig * $scaleFactor)); |
317
|
|
|
|
318
|
3 |
|
$success = $image->preciseResize($newWidth, $newHeight); |
319
|
3 |
|
if ($success === false) { |
320
|
|
|
throw new \RuntimeException("Error during image resize"); |
321
|
|
|
} |
322
|
|
|
|
323
|
3 |
|
$this->logDebug(sprintf('Image scaled from %dx%d to %dx%d (since max image area is %d pixels^2)', |
324
|
3 |
|
$widthOrig, $heightOrig, $newWidth, $newHeight, $maxImageArea)); |
325
|
|
|
|
326
|
3 |
|
return 1 / $scaleFactor; |
327
|
|
|
} |
328
|
|
|
|
329
|
|
|
/** |
330
|
|
|
* Gets all face descriptors in a given image processing context. Populates "descriptor" in array of faces. |
331
|
|
|
* |
332
|
|
|
* @param IModel $model Resnet model |
333
|
|
|
* @param ImageProcessingContext Image processing context |
334
|
|
|
*/ |
335
|
2 |
|
private function populateDescriptors(IModel $model, ImageProcessingContext $imageProcessingContext) { |
336
|
2 |
|
$faces = $imageProcessingContext->getFaces(); |
337
|
|
|
|
338
|
2 |
|
foreach($faces as &$face) { |
339
|
|
|
// For each face, we want to detect landmarks and compute descriptors. |
340
|
|
|
// We use already resized image (from temp, used to detect faces) for this. |
341
|
|
|
// (better would be to work with original image, but that will require |
342
|
|
|
// another orientation fix and another save to the temp) |
343
|
|
|
// But, since our face coordinates are already changed to align to original image, |
344
|
|
|
// we need to fix them up to align them to temp image here. |
345
|
1 |
|
$normalizedFace = clone $face; |
346
|
1 |
|
$normalizedFace->normalizeSize(1.0 / $imageProcessingContext->getRatio()); |
347
|
|
|
|
348
|
|
|
// We are getting face landmarks from already prepared (temp) image (resized and with orienation fixed). |
349
|
1 |
|
$landmarks = $model->detectLandmarks($imageProcessingContext->getTempPath(), array( |
350
|
1 |
|
"left" => $normalizedFace->left, "top" => $normalizedFace->top, |
351
|
1 |
|
"bottom" => $normalizedFace->bottom, "right" => $normalizedFace->right)); |
352
|
1 |
|
$face->landmarks = $landmarks['parts']; |
353
|
|
|
|
354
|
1 |
|
$descriptor = $model->computeDescriptor($imageProcessingContext->getTempPath(), $landmarks); |
355
|
1 |
|
$face->descriptor = $descriptor; |
356
|
|
|
} |
357
|
2 |
|
} |
358
|
|
|
|
359
|
|
|
/** |
360
|
|
|
* Obtains max image area lazily (from cache, or calculates it and puts it to cache) |
361
|
|
|
* |
362
|
|
|
* @return int Max image area (in pixels^2) |
363
|
|
|
*/ |
364
|
2 |
|
private function getMaxImageArea(): int { |
365
|
|
|
// First check if is cached |
366
|
|
|
// |
367
|
2 |
|
if (!is_null($this->maxImageAreaCached)) { |
368
|
|
|
return $this->maxImageAreaCached; |
369
|
|
|
} |
370
|
|
|
|
371
|
|
|
// Get this setting on main app_config. |
372
|
|
|
// Note that this option has lower and upper limits and validations |
373
|
2 |
|
$this->maxImageAreaCached = $this->settingsService->getAnalysisImageArea(); |
374
|
|
|
|
375
|
|
|
// Check if admin override it in config and it is valid value |
376
|
|
|
// |
377
|
2 |
|
$maxImageArea = $this->settingsService->getMaximumImageArea(); |
378
|
2 |
|
if ($maxImageArea > 0) { |
379
|
2 |
|
$this->maxImageAreaCached = $maxImageArea; |
380
|
|
|
} |
381
|
|
|
// Also check if we are provided value from command line. |
382
|
|
|
// |
383
|
2 |
|
if ((array_key_exists('max_image_area', $this->context->propertyBag)) && |
384
|
2 |
|
(!is_null($this->context->propertyBag['max_image_area']))) { |
385
|
|
|
$this->maxImageAreaCached = $this->context->propertyBag['max_image_area']; |
386
|
|
|
} |
387
|
|
|
|
388
|
2 |
|
return $this->maxImageAreaCached; |
389
|
|
|
} |
390
|
|
|
|
391
|
|
|
} |