OCA\FaceRecognition\BackgroundJob\Tasks\ImageProcessingTask

Complexity

Total Complexity

24

Size/Duplication

Total Lines	249
Duplicated Lines	0 %

Test Coverage

Coverage

86.92%

Importance

Changes	2
Bugs	0	Features	0

7 Methods

Rating	Name	Duplication	Size	Complexity
A	getMaxImageArea()	0	25	5
A	__construct()	0	16	1
A	getTempImage()	0	23	4
A	getNormalizedFace()	0	10	1
A	description()	0	2	1
D	execute()	0	97	10
A	getNormalizedLandmarks()	0	9	2

<?php
/**
 * @copyright Copyright (c) 2017-2020 Matias De lellis <mati86dl@gmail.com>
 * @copyright Copyright (c) 2018, Branko Kokanovic <branko@kokanovic.org>
 *
 * @author Branko Kokanovic <branko@kokanovic.org>
 *
 * @license GNU AGPL version 3 or any later version
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
namespace OCA\FaceRecognition\BackgroundJob\Tasks;

use OCP\Image as OCP_Image;

use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Lock\ILockingProvider;
use OCP\IUser;

use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask;
use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext;

use OCA\FaceRecognition\Db\Face;
use OCA\FaceRecognition\Db\Image;
use OCA\FaceRecognition\Db\ImageMapper;

use OCA\FaceRecognition\Helper\TempImage;

use OCA\FaceRecognition\Model\IModel;
use OCA\FaceRecognition\Model\ModelManager;

use OCA\FaceRecognition\Service\FileService;
use OCA\FaceRecognition\Service\SettingsService;

/**
 * Taks that get all images that are still not processed and processes them.
 * Processing image means that each image is prepared, faces extracted form it,
 * and for each found face - face descriptor is extracted.
 */
class ImageProcessingTask extends FaceRecognitionBackgroundTask {

	/** @var ImageMapper Image mapper*/
	protected $imageMapper;

	/** @var FileService */
	protected $fileService;

	/** @var SettingsService */
	protected $settingsService;

	/** @var ModelManager $modelManager */
	protected $modelManager;

	/** @var ILockingProvider $lockingProvider */
	protected ILockingProvider $lockingProvider;

	/** @var IModel $model */
	private $model;

	/** @var int|null $maxImageAreaCached Maximum image area (cached, so it is not recalculated for each image) */
	private $maxImageAreaCached;


	/**
	 * @param ImageMapper $imageMapper Image mapper
	 * @param FileService $fileService
	 * @param SettingsService $settingsService
	 * @param ModelManager $modelManager Model manager
	 * @param ILockingProvider $lockingProvider
	 */
	public function __construct(ImageMapper      $imageMapper,
	                            FileService      $fileService,
	                            SettingsService  $settingsService,
	                            ModelManager     $modelManager,
	                            ILockingProvider $lockingProvider)
	{
		parent::__construct();

		$this->imageMapper        = $imageMapper;
		$this->fileService        = $fileService;
		$this->settingsService    = $settingsService;
		$this->modelManager       = $modelManager;
		$this->lockingProvider    = $lockingProvider;

		$this->model              = null;
		$this->maxImageAreaCached = null;
	}

	/**
	 * @inheritdoc
	 */
	public function description() {
		return "Process all images to extract faces";
	}

	/**
	 * @inheritdoc
	 */
	public function execute(FaceRecognitionContext $context) {
		$this->setContext($context);

		$this->logInfo('NOTE: Starting face recognition. If you experience random crashes after this point, please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ');

		// Get current model.
		$this->model = $this->modelManager->getCurrentModel();

		// Open model.
		$this->model->open();

		$images = $context->propertyBag['images'];
		foreach($images as $image) {
			yield;

			$startMillis = round(microtime(true) * 1000);

			try {
				// Get a image lock
				$lockKey = 'facerecognition/' . $image->getId();
				$lockType = ILockingProvider::LOCK_EXCLUSIVE;
				$this->lockingProvider->acquireLock($lockKey, $lockType);

				$dbImage = $this->imageMapper->find($image->getUser(), $image->getId());
				if ($dbImage->getIsProcessed()) {
					$this->logInfo('Faces found: 0. Image will be skipped since it was already processed.');
					// Release lock of file.
					$this->lockingProvider->releaseLock($lockKey, $lockType);
					continue;
				}


				// Get an temp Image to process this image.
				$tempImage = $this->getTempImage($image);

				if (is_null($tempImage)) {
					// If we cannot find a file probably it was deleted out of our control and we must clean our tables.
					$this->settingsService->setNeedRemoveStaleImages(true, $image->user);
					$this->logInfo('File with ID ' . $image->file . ' doesn\'t exist anymore, skipping it');
					// Release lock of file.
					$this->lockingProvider->releaseLock($lockKey, $lockType);
					continue;
				}

				if ($tempImage->getSkipped() === true) {
					$this->logInfo('Faces found: 0 (image will be skipped because it is too small)');
					$this->imageMapper->imageProcessed($image, array(), 0);
					// Release lock of file.
					$this->lockingProvider->releaseLock($lockKey, $lockType);
					continue;
				}

				// Get faces in the temporary image
				$tempImagePath = $tempImage->getTempPath();
				$rawFaces = $this->model->detectFaces($tempImagePath);

				$this->logInfo('Faces found: ' . count($rawFaces));

				$faces = array();
				foreach ($rawFaces as $rawFace) {
					// Normalize face and landmarks from model to original size
					$normFace = $this->getNormalizedFace($rawFace, $tempImage->getRatio());
					// Convert from dictionary of face to our Face Db Entity.
					$face = Face::fromModel($image->getId(), $normFace);
					// Save the normalized Face to insert on database later.
					$faces[] = $face;
				}

				// Save new faces fo database
				$endMillis = round(microtime(true) * 1000);
				$duration = (int) max($endMillis - $startMillis, 0);
				$this->imageMapper->imageProcessed($image, $faces, $duration);

				// Release lock of file.
				$this->lockingProvider->releaseLock($lockKey, $lockType);
			} catch (\OCP\Lock\LockedException $e) {
				$this->logInfo('Faces found: 0. Image will be skipped because it is locked');
			} catch (\Exception $e) {
				if ($e->getMessage() === "std::bad_alloc") {
					throw new \RuntimeException("Not enough memory to run face recognition! Please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ");
				}
				$this->logInfo('Faces found: 0. Image will be skipped because of the following error: ' . $e->getMessage());
				$this->logDebug((string) $e);

				// Save an empty entry so it can be analyzed again later
				$this->imageMapper->imageProcessed($image, array(), 0, $e);
			} finally {
				// Clean temporary image.
				if (isset($tempImage)) {
					$tempImage->clean();
				}
				// If there are temporary files from external files, they must also be cleaned.
				$this->fileService->clean();
			}
		}

		return true;
	}

	/**
	 * Given an image, build a temporary image to perform the analysis
	 *
	 * return TempImage|null
	 */
	private function getTempImage(Image $image): ?TempImage {
		// todo: check if this hits I/O (database, disk...), consider having lazy caching to return user folder from user
		$file = $this->fileService->getFileById($image->getFile(), $image->getUser());
		if (empty($file)) {
			return null;
		}

		if (!$this->fileService->isAllowedNode($file)) {
			return null;
		}

		$imagePath = $this->fileService->getLocalFile($file);
		if ($imagePath === null)
			return null;

		$this->logInfo('Processing image ' . $imagePath);

		$tempImage = new TempImage($imagePath,
		                           $this->model->getPreferredMimeType(),
		                           $this->getMaxImageArea(),
		                           $this->settingsService->getMinimumImageSize());

		return $tempImage;
	}

	/**
	 * Obtains max image area lazily (from cache, or calculates it and puts it to cache)
	 *
	 * @return int Max image area (in pixels^2)
	 */
	private function getMaxImageArea(): int {
		// First check if is cached
		//
		if (!is_null($this->maxImageAreaCached)) {
			return $this->maxImageAreaCached;
		}

		// Get this setting on main app_config.
		// Note that this option has lower and upper limits and validations
		$this->maxImageAreaCached = $this->settingsService->getAnalysisImageArea();

		// Check if admin override it in config and it is valid value
		//
		$maxImageArea = $this->settingsService->getMaximumImageArea();
		if ($maxImageArea > 0) {
			$this->maxImageAreaCached = $maxImageArea;
		}
		// Also check if we are provided value from command line.
		//
		if ((array_key_exists('max_image_area', $this->context->propertyBag)) &&
		    (!is_null($this->context->propertyBag['max_image_area']))) {
			$this->maxImageAreaCached = $this->context->propertyBag['max_image_area'];
		}

		return $this->maxImageAreaCached;
	}

	/**
	 * Helper method, to normalize face sizes back to original dimensions, based on ratio
	 *
	 */
	private function getNormalizedFace(array $rawFace, float $ratio): array {
		$face = [];
		$face['left'] = intval(round($rawFace['left']*$ratio));
		$face['right'] = intval(round($rawFace['right']*$ratio));
		$face['top'] = intval(round($rawFace['top']*$ratio));
		$face['bottom'] = intval(round($rawFace['bottom']*$ratio));
		$face['detection_confidence'] = $rawFace['detection_confidence'];
		$face['landmarks'] = $this->getNormalizedLandmarks($rawFace['landmarks'], $ratio);
		$face['descriptor'] = $rawFace['descriptor'];
		return $face;
	}

	/**
	 * Helper method, to normalize landmarks sizes back to original dimensions, based on ratio
	 *
	 */
	private function getNormalizedLandmarks(array $rawLandmarks, float $ratio): array {
		$landmarks = [];
		foreach ($rawLandmarks as $rawLandmark) {
			$landmark = [];
			$landmark['x'] = intval(round($rawLandmark['x']*$ratio));
			$landmark['y'] = intval(round($rawLandmark['y']*$ratio));
			$landmarks[] = $landmark;
		}
		return $landmarks;
	}

}