1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* @copyright Copyright (c) 2017, Matias De lellis <[email protected]> |
4
|
|
|
* @copyright Copyright (c) 2018, Branko Kokanovic <[email protected]> |
5
|
|
|
* |
6
|
|
|
* @author Branko Kokanovic <[email protected]> |
7
|
|
|
* |
8
|
|
|
* @license GNU AGPL version 3 or any later version |
9
|
|
|
* |
10
|
|
|
* This program is free software: you can redistribute it and/or modify |
11
|
|
|
* it under the terms of the GNU Affero General Public License as |
12
|
|
|
* published by the Free Software Foundation, either version 3 of the |
13
|
|
|
* License, or (at your option) any later version. |
14
|
|
|
* |
15
|
|
|
* This program is distributed in the hope that it will be useful, |
16
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
17
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
18
|
|
|
* GNU Affero General Public License for more details. |
19
|
|
|
* |
20
|
|
|
* You should have received a copy of the GNU Affero General Public License |
21
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
22
|
|
|
* |
23
|
|
|
*/ |
24
|
|
|
namespace OCA\FaceRecognition\BackgroundJob\Tasks; |
25
|
|
|
|
26
|
|
|
use OC_Image; |
27
|
|
|
|
28
|
|
|
use OCP\Files\File; |
29
|
|
|
use OCP\Files\Folder; |
30
|
|
|
use OCP\IConfig; |
31
|
|
|
use OCP\ITempManager; |
32
|
|
|
use OCP\IUser; |
33
|
|
|
|
34
|
|
|
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask; |
35
|
|
|
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext; |
36
|
|
|
use OCA\FaceRecognition\Db\FaceNew; |
37
|
|
|
use OCA\FaceRecognition\Db\Image; |
38
|
|
|
use OCA\FaceRecognition\Db\ImageMapper; |
39
|
|
|
use OCA\FaceRecognition\Helper\Requirements; |
40
|
|
|
use OCA\FaceRecognition\Migration\AddDefaultFaceModel; |
41
|
|
|
|
42
|
|
|
/** |
43
|
|
|
* Plain old PHP object holding all information |
44
|
|
|
* that are needed to process all faces from one image |
45
|
|
|
*/ |
46
|
|
|
class ImageProcessingContext { |
47
|
|
|
/** @var string Path to the image being processed */ |
48
|
|
|
private $imagePath; |
49
|
|
|
|
50
|
|
|
/** @var string Path to temporary, resized image */ |
51
|
|
|
private $tempPath; |
52
|
|
|
|
53
|
|
|
/** @var float Ratio of resized image, when scaling it */ |
54
|
|
|
private $ratio; |
55
|
|
|
|
56
|
|
|
/** @var array<FaceNew> All found faces in image */ |
57
|
|
|
private $faces; |
58
|
|
|
|
59
|
|
|
public function __construct(string $imagePath, string $tempPath, float $ratio) { |
60
|
|
|
$this->imagePath = $imagePath; |
61
|
|
|
$this->tempPath = $tempPath; |
62
|
|
|
$this->ratio = $ratio; |
63
|
|
|
} |
64
|
|
|
|
65
|
|
|
public function getImagePath(): string { |
66
|
|
|
return $this->imagePath; |
67
|
|
|
} |
68
|
|
|
|
69
|
|
|
public function getTempPath(): string { |
70
|
|
|
return $this->tempPath; |
71
|
|
|
} |
72
|
|
|
|
73
|
|
|
public function getRatio(): float { |
74
|
|
|
return $this->ratio; |
75
|
|
|
} |
76
|
|
|
|
77
|
|
|
/** |
78
|
|
|
* Gets all faces |
79
|
|
|
* |
80
|
|
|
* @return FaceNew[] Array of faces |
81
|
|
|
*/ |
82
|
|
|
public function getFaces(): array { |
83
|
|
|
return $this->faces; |
84
|
|
|
} |
85
|
|
|
|
86
|
|
|
/** |
87
|
|
|
* @param array<FaceNew> $faces Array of faces to set |
88
|
|
|
*/ |
89
|
|
|
public function setFaces($faces) { |
90
|
|
|
$this->faces = $faces; |
91
|
|
|
} |
92
|
|
|
} |
93
|
|
|
|
94
|
|
|
/** |
95
|
|
|
* Taks that get all images that are still not processed and processes them. |
96
|
|
|
* Processing image means that each image is prepared, faces extracted form it, |
97
|
|
|
* and for each found face - face descriptor is extracted. |
98
|
|
|
*/ |
99
|
|
|
class ImageProcessingTask extends FaceRecognitionBackgroundTask { |
100
|
|
|
/** @var IConfig Config */ |
101
|
|
|
private $config; |
102
|
|
|
|
103
|
|
|
/** @var ImageMapper Image mapper*/ |
104
|
|
|
protected $imageMapper; |
105
|
|
|
|
106
|
|
|
/** @var ITempManager */ |
107
|
|
|
private $tempManager; |
108
|
|
|
|
109
|
|
|
/** |
110
|
|
|
* @param ImageMapper $imageMapper Image mapper |
111
|
|
|
*/ |
112
|
|
|
public function __construct(IConfig $config, ImageMapper $imageMapper, ITempManager $tempManager) { |
113
|
|
|
parent::__construct(); |
114
|
|
|
$this->config = $config; |
115
|
|
|
$this->imageMapper = $imageMapper; |
116
|
|
|
$this->tempManager = $tempManager; |
117
|
|
|
} |
118
|
|
|
|
119
|
|
|
/** |
120
|
|
|
* @inheritdoc |
121
|
|
|
*/ |
122
|
|
|
public function description() { |
123
|
|
|
return "Process all images to extract faces"; |
124
|
|
|
} |
125
|
|
|
|
126
|
|
|
/** |
127
|
|
|
* @inheritdoc |
128
|
|
|
*/ |
129
|
|
|
public function execute(FaceRecognitionContext $context) { |
130
|
|
|
$this->setContext($context); |
131
|
|
|
|
132
|
|
|
$model = intval($this->config->getAppValue('facerecognition', 'model', AddDefaultFaceModel::DEFAULT_FACE_MODEL_ID)); |
133
|
|
|
$requirements = new Requirements($context->appManager, $model); |
134
|
|
|
|
135
|
|
|
$dataDir = rtrim($context->config->getSystemValue('datadirectory', \OC::$SERVERROOT.'/data'), '/'); |
136
|
|
|
$images = $context->propertyBag['images']; |
137
|
|
|
|
138
|
|
|
$cfd = new \CnnFaceDetection($requirements->getFaceDetectionModelv2()); |
139
|
|
|
$fld = new \FaceLandmarkDetection($requirements->getLandmarksDetectionModelv2()); |
140
|
|
|
$fr = new \FaceRecognition($requirements->getFaceRecognitionModelv2()); |
141
|
|
|
|
142
|
|
|
foreach($images as $image) { |
143
|
|
|
yield; |
144
|
|
|
|
145
|
|
|
$imageProcessingContext = null; |
146
|
|
|
$startMillis = round(microtime(true) * 1000); |
147
|
|
|
try { |
148
|
|
|
$imageProcessingContext = $this->findFaces($cfd, $dataDir, $image); |
149
|
|
|
if ($imageProcessingContext == null) { |
150
|
|
|
// We didn't got exception, but null result means we should skip this image |
151
|
|
|
continue; |
152
|
|
|
} |
153
|
|
|
|
154
|
|
|
$this->populateDescriptors($fld, $fr, $imageProcessingContext); |
155
|
|
|
|
156
|
|
|
$endMillis = round(microtime(true) * 1000); |
157
|
|
|
$duration = max($endMillis - $startMillis, 0); |
158
|
|
|
$this->imageMapper->imageProcessed($image, $imageProcessingContext->getFaces(), $duration); |
159
|
|
|
} catch (\Exception $e) { |
160
|
|
|
$this->imageMapper->imageProcessed($image, array(), 0, $e); |
161
|
|
|
} finally { |
162
|
|
|
$this->tempManager->clean(); |
163
|
|
|
} |
164
|
|
|
} |
165
|
|
|
|
166
|
|
|
return true; |
|
|
|
|
167
|
|
|
} |
168
|
|
|
|
169
|
|
|
/** |
170
|
|
|
* Given an image, it finds all faces on it. |
171
|
|
|
* If image should be skipped, returns null. |
172
|
|
|
* If there is any error, throws exception |
173
|
|
|
* |
174
|
|
|
* @param \CnnFaceDetection $cfd Face detection model |
175
|
|
|
* @param string $dataDir Directory where data is stored |
176
|
|
|
* @param Image $image Image to find faces on |
177
|
|
|
* @return ImageProcessingContext|null Generated context that hold all information needed later for this image |
178
|
|
|
*/ |
179
|
|
|
private function findFaces(\CnnFaceDetection $cfd, string $dataDir, Image $image) { |
180
|
|
|
// todo: check if this hits I/O (database, disk...), consider having lazy caching to return user folder from user |
181
|
|
|
$userFolder = $this->context->rootFolder->getUserFolder($image->user); |
182
|
|
|
$userRoot = $userFolder->getParent(); |
183
|
|
|
$file = $userRoot->getById($image->file); |
184
|
|
|
if (empty($file)) { |
185
|
|
|
$this->logInfo('File with ID ' . $image->file . ' doesn\'t exist anymore, skipping it'); |
186
|
|
|
return null; |
187
|
|
|
} |
188
|
|
|
|
189
|
|
|
// todo: this concat is wrong with shared files. |
190
|
|
|
$imagePath = $dataDir . $file[0]->getPath(); |
191
|
|
|
$this->logInfo('Processing image ' . $imagePath); |
192
|
|
|
$imageProcessingContext = $this->prepareImage($imagePath); |
193
|
|
|
|
194
|
|
|
// Detect faces from model |
195
|
|
|
$facesFound = $cfd->detect($imageProcessingContext->getTempPath()); |
196
|
|
|
|
197
|
|
|
// Convert from dictionary of faces to our Face Db Entity |
198
|
|
|
$faces = array(); |
199
|
|
|
foreach ($facesFound as $faceFound) { |
200
|
|
|
$face = FaceNew::fromModel($image, $faceFound); |
201
|
|
|
$face->normalizeSize($imageProcessingContext->getRatio()); |
202
|
|
|
$faces[] = $face; |
203
|
|
|
} |
204
|
|
|
|
205
|
|
|
$imageProcessingContext->setFaces($faces); |
206
|
|
|
$this->logInfo('Faces found ' . count($faces)); |
207
|
|
|
|
208
|
|
|
return $imageProcessingContext; |
209
|
|
|
} |
210
|
|
|
|
211
|
|
|
/** |
212
|
|
|
* Given an image, it will rotate, scale and save image to temp location, ready to be consumed by pdlib. |
213
|
|
|
* |
214
|
|
|
* @param string $imagePath Path to image on disk |
215
|
|
|
* |
216
|
|
|
* @return ImageProcessingContext Generated context that hold all information needed later for this image |
217
|
|
|
*/ |
218
|
|
|
private function prepareImage(string $imagePath): ImageProcessingContext { |
219
|
|
|
$image = new \OC_Image(null, $this->context->logger->getLogger(), $this->context->config); |
220
|
|
|
$image->loadFromFile($imagePath); |
221
|
|
|
$image->fixOrientation(); |
222
|
|
|
if (!$image->valid()) { |
223
|
|
|
throw new \RuntimeException("Image is not valid, probably cannot be loaded"); |
224
|
|
|
} |
225
|
|
|
|
226
|
|
|
// todo: be smarter with this 1024 constant. Depending on GPU/memory of the host, this can be larger. |
227
|
|
|
$ratio = $this->resizeImage($image, 1024); |
228
|
|
|
|
229
|
|
|
$tempfile = $this->tempManager->getTemporaryFile(pathinfo($imagePath, PATHINFO_EXTENSION)); |
230
|
|
|
$image->save($tempfile); |
231
|
|
|
return new ImageProcessingContext($imagePath, $tempfile, $ratio); |
232
|
|
|
} |
233
|
|
|
|
234
|
|
|
/** |
235
|
|
|
* Resizes the image preserving ratio. Stolen and adopted from OC_Image->resize(). |
236
|
|
|
* Difference is that this returns ratio of resize. |
237
|
|
|
* Also, resize is not done if $maxSize is less than both width and height. |
238
|
|
|
* |
239
|
|
|
* @* @param OC_Image $image Image to resize |
240
|
|
|
* @param int $maxSize The maximum size of either the width or height. |
241
|
|
|
* @return float Ratio of resize. 1 if there was no resize |
242
|
|
|
*/ |
243
|
|
|
public function resizeImage(OC_Image $image, int $maxSize): float { |
244
|
|
|
if (!$image->valid()) { |
245
|
|
|
$this->logInfo(__METHOD__ . '(): No image loaded', array('app' => 'core')); |
|
|
|
|
246
|
|
|
throw new \RuntimeException("Image is not valid, probably cannot be loaded"); |
247
|
|
|
} |
248
|
|
|
|
249
|
|
|
$widthOrig = imagesx($image->resource()); |
250
|
|
|
$heightOrig = imagesy($image->resource()); |
251
|
|
|
if (($widthOrig < $maxSize) && ($heightOrig < $maxSize)) { |
252
|
|
|
return 1.0; |
253
|
|
|
} |
254
|
|
|
|
255
|
|
|
$ratioOrig = $widthOrig / $heightOrig; |
256
|
|
|
|
257
|
|
|
if ($ratioOrig > 1) { |
258
|
|
|
$newHeight = round($maxSize / $ratioOrig); |
259
|
|
|
$newWidth = $maxSize; |
260
|
|
|
} else { |
261
|
|
|
$newWidth = round($maxSize * $ratioOrig); |
262
|
|
|
$newHeight = $maxSize; |
263
|
|
|
} |
264
|
|
|
|
265
|
|
|
$success = $image->preciseResize((int)round($newWidth), (int)round($newHeight)); |
266
|
|
|
if ($success == false) { |
267
|
|
|
throw new \RuntimeException("Error during image resize"); |
268
|
|
|
} |
269
|
|
|
|
270
|
|
|
return $widthOrig / $newWidth; |
271
|
|
|
} |
272
|
|
|
|
273
|
|
|
/** |
274
|
|
|
* Gets all face descriptors in a given image processing context. Populates "descriptor" in array of faces. |
275
|
|
|
* |
276
|
|
|
* @param \FaceLandmarkDetection $fld Landmark detection model |
277
|
|
|
* @param \FaceRecognition $fr Face recognition model |
278
|
|
|
* @param ImageProcessingContext Image processing context |
279
|
|
|
*/ |
280
|
|
|
private function populateDescriptors(\FaceLandmarkDetection $fld, \FaceRecognition $fr, ImageProcessingContext $imageProcessingContext) { |
281
|
|
|
$faces = $imageProcessingContext->getFaces(); |
282
|
|
|
|
283
|
|
|
foreach($faces as &$face) { |
284
|
|
|
$tempfilePath = $this->cropFace($imageProcessingContext->getImagePath(), $face); |
285
|
|
|
|
286
|
|
|
// Usually, second argument to detect should be just $face. However, since we are doing image acrobatics |
287
|
|
|
// and already have cropped image, bounding box for landmark detection is now complete (cropped) image! |
288
|
|
|
$landmarks = $fld->detect($tempfilePath, array( |
289
|
|
|
"left" => 0, "top" => 0, "bottom" => $face->height(), "right" => $face->width())); |
290
|
|
|
$descriptor = $fr->computeDescriptor($tempfilePath, $landmarks); |
291
|
|
|
$face->descriptor = $descriptor; |
292
|
|
|
} |
293
|
|
|
} |
294
|
|
|
|
295
|
|
|
private function cropFace(string $imagePath, FaceNew $face): string { |
296
|
|
|
// todo: we are loading same image two times, fix this |
297
|
|
|
$image = new \OC_Image(null, $this->context->logger->getLogger(), $this->context->config); |
298
|
|
|
$image->loadFromFile($imagePath); |
299
|
|
|
$image->fixOrientation(); |
300
|
|
|
$success = $image->crop($face->left, $face->top, $face->width(), $face->height()); |
301
|
|
|
if ($success == false) { |
302
|
|
|
throw new \RuntimeException("Error during image cropping"); |
303
|
|
|
} |
304
|
|
|
|
305
|
|
|
$tempfile = $this->tempManager->getTemporaryFile(pathinfo($imagePath, PATHINFO_EXTENSION)); |
306
|
|
|
$image->save($tempfile); |
307
|
|
|
return $tempfile; |
308
|
|
|
} |
309
|
|
|
} |
If you return a value from a function or method, it should be a sub-type of the type that is given by the parent type f.e. an interface, or abstract method. This is more formally defined by the Lizkov substitution principle, and guarantees that classes that depend on the parent type can use any instance of a child type interchangably. This principle also belongs to the SOLID principles for object oriented design.
Let’s take a look at an example:
Our function
my_function
expects aPost
object, and outputs the author of the post. The base classPost
returns a simple string and outputting a simple string will work just fine. However, the child classBlogPost
which is a sub-type ofPost
instead decided to return anobject
, and is therefore violating the SOLID principles. If aBlogPost
were passed tomy_function
, PHP would not complain, but ultimately fail when executing thestrtoupper
call in its body.