ImageExtractor::buildImagePath() - Code Metrics - Inspection of "Extract images: add part of article url path when..." - scotteh/php-goose - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 84b1b9...90721d )

by Andrew

created 2017-12-19 10:44 UTC

ImageExtractor::buildImagePath() D

↳ Parent: ImageExtractor

Complexity

Conditions	9
Paths	8

Size

Total Lines	27
Code Lines	18

Duplication

Lines	0
Ratio	0 %

Importance

Changes	3
Bugs	0	Features	1

Metric	Value
c	3
b	0
f	1
dl	0
loc	27
rs	4.909
cc	9
eloc	18
nc	8
nop	1

<?php

namespace Goose\Modules\Extractors;

use Goose\Article;
use Goose\Images\Image;
use Goose\Images\ImageUtils;
use Goose\Images\LocallyStoredImage;
use Goose\Traits\ArticleMutatorTrait;
use Goose\Modules\AbstractModule;
use Goose\Modules\ModuleInterface;
use DOMWrap\Element;
use DOMWrap\NodeList;

/**
 * Image Extractor
 *
 * @package Goose\Modules\Extractors
 * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License 2.0
 */
class ImageExtractor extends AbstractModule implements ModuleInterface {
    use ArticleMutatorTrait;

    /** @var string[] */
    private $badFileNames = [
        '\.html', '\.gif', '\.ico', 'button', 'twitter\.jpg', 'facebook\.jpg',
        'ap_buy_photo', 'digg\.jpg', 'digg\.png', 'delicious\.png',
        'facebook\.png', 'reddit\.jpg', 'doubleclick', 'diggthis',
        'diggThis', 'adserver', '\/ads\/', 'ec\.atdmt\.com', 'mediaplex\.com',
        'adsatt', 'view\.atdmt',
    ];

    /** @var string[] */
    private static $KNOWN_IMG_DOM_NAMES = [
        'yn-story-related-media',
        'cnn_strylccimg300cntr',
        'big_photo',
        'ap-smallphoto-a'
    ];

    /** @var int */
    private static $MAX_PARENT_DEPTH = 2;

    /** @var string[] */
    private static $CUSTOM_SITE_MAPPING = [];

    /**
     * @param Article $article
     */
    public function run(Article $article) {
        $this->article($article);

        if ($this->config()->get('image_fetch_best')) {
            $article->setTopImage($this->getBestImage());

            if ($this->config()->get('image_fetch_all')
              && $article->getTopNode() instanceof Element) {
                $article->setAllImages($this->getAllImages());
            }
        }
    }

    /**
     * @return Image|null
     */
    private function getBestImage() {
        $image = $this->checkForKnownElements();

        if ($image) {
            return $image;
        }

        $image = $this->checkForMetaTag();

        if ($image) {
            return $image;
        }

        if ($this->article()->getTopNode() instanceof Element) {
            $image = $this->checkForLargeImages($this->article()->getTopNode(), 0, 0);

            if ($image) {
                return $image;
            }
        }

        return null;
    }

    /**
     * Prefer Twitter images (as they tend to have the right size for us), then Open Graph images
     * (which seem to be smaller), and finally linked images.
     *
     * @return Image|null
     */
    private function checkForMetaTag() {
        $image = $this->checkForTwitterTag();

        if ($image) {
            return $image;
        }

        $image = $this->checkForOpenGraphTag();

        if ($image) {
            return $image;
        }

        $image = $this->checkForLinkTag();

        if ($image) {
            return $image;
        }

        return null;
    }

    /**
     * although slow the best way to determine the best image is to download them and check the actual dimensions of the image when on disk
     * so we'll go through a phased approach...
     * 1. get a list of ALL images from the parent node
     * 2. filter out any bad image names that we know of (gifs, ads, etc..)
     * 3. do a head request on each file to make sure it meets our bare requirements
     * 4. any images left over let's do a full GET request, download em to disk and check their dimensions
     * 5. Score images based on different factors like height/width and possibly things like color density
     *
     * @param Element $node
     * @param int $parentDepthLevel
     * @param int $siblingDepthLevel
     *
     * @return Image|null
     */
    private function checkForLargeImages(Element $node, $parentDepthLevel, $siblingDepthLevel) {
        $goodLocalImages = $this->getImageCandidates($node);

        $scoredLocalImages = $this->scoreLocalImages($goodLocalImages, $parentDepthLevel);

        ksort($scoredLocalImages);

        if (!empty($scoredLocalImages)) {
            foreach ($scoredLocalImages as $imageScore => $scoredLocalImage) {
                $mainImage = new Image();
                $mainImage->setImageSrc($scoredLocalImage->getImgSrc());
                $mainImage->setImageExtractionType('bigimage');
                $mainImage->setConfidenceScore(100 / count($scoredLocalImages));
                $mainImage->setImageScore($imageScore);
                $mainImage->setBytes($scoredLocalImage->getBytes());
                $mainImage->setHeight($scoredLocalImage->getHeight());
                $mainImage->setWidth($scoredLocalImage->getWidth());

                return $mainImage;
            }
        } else {
            $depthObj = $this->getDepthLevel($node, $parentDepthLevel, $siblingDepthLevel);

            if ($depthObj && NULL !== $depthObj->node) {
                return $this->checkForLargeImages($depthObj->node, $depthObj->parentDepth, $depthObj->siblingDepth);
            }
        }

        return null;
    }

    /**
     * @param Element $node
     * @param int $parentDepth
     * @param int $siblingDepth
     *
     * @return object|null
     */
    private function getDepthLevel(Element $node, $parentDepth, $siblingDepth) {
        if (is_null($node) || !($node->parent() instanceof Element)) {
            return null;
        }

        if ($parentDepth > self::$MAX_PARENT_DEPTH) {
            return null;
        }

        // Find previous sibling element node
        $siblingNode = $node->preceding(function($node) {
            return $node instanceof Element;
        });

        if (is_null($siblingNode)) {
            return (object)[
                'node' => $node->parent(),
                'parentDepth' => $parentDepth + 1,
                'siblingDepth' => 0,
            ];
        }

        return (object)[
            'node' => $siblingNode,
            'parentDepth' => $parentDepth,
            'siblingDepth' => $siblingDepth + 1,
        ];
    }

    /**
     * Set image score and on locally downloaded images
     *
     * we're going to score the images in the order in which they appear so images higher up will have more importance,
     * we'll count the area of the 1st image as a score of 1 and then calculate how much larger or small each image after it is
     * we'll also make sure to try and weed out banner type ad blocks that have big widths and small heights or vice versa
     * so if the image is 3rd found in the dom it's sequence score would be 1 / 3 = .33 * diff in area from the first image
     *
     * @param LocallyStoredImage[] $locallyStoredImages
     * @param int $depthLevel
     *
     * @return LocallyStoredImage[]
     */
    private function scoreLocalImages($locallyStoredImages, $depthLevel) {

        $results = [];
        $i = 1;
        $initialArea = 0;

        // Limit to the first 30 images
        $locallyStoredImages = array_slice($locallyStoredImages, 0, 30);

        foreach ($locallyStoredImages as $locallyStoredImage) {
            $sequenceScore = 1 / $i;
            $area = $locallyStoredImage->getWidth() * $locallyStoredImage->getHeight();

            if ($initialArea == 0) {
                $initialArea = $area * 1.48;
                $totalScore = 1;
            } else {
                $areaDifference = $area * $initialArea;
                $totalScore = $sequenceScore * $areaDifference;
            }

            $i++;

            $results[$totalScore] = $locallyStoredImage;
        }

        return $results;
    }

    /**
     * @param LocallyStoredImage $locallyStoredImage
     * @param int $depthLevel
     *
     * @return bool
     */
    private function isWorthyImage($locallyStoredImage, $depthLevel) {

        if ($locallyStoredImage->getWidth() <= $this->config()->get('image_min_width')
          || $locallyStoredImage->getHeight() <= $this->config()->get('image_min_height')
          || $locallyStoredImage->getFileExtension() == 'NA'
          || ($depthLevel < 1 && $locallyStoredImage->getWidth() < 300) || $depthLevel >= 1
          || $this->isBannerDimensions($locallyStoredImage->getWidth(), $locallyStoredImage->getHeight())) {
            return false;
        }

        return true;
    }

    /**
     * @return Image[]
     */
    private function getAllImages() {
        $results = [];

        $images = $this->article()->getTopNode()->find('img');

        // Generate a complete URL for each image
        $imageUrls = array_map(function($image) {
            return $this->buildImagePath($image->attr('src'));
        }, $images->toArray());

        $localImages = $this->getLocallyStoredImages($imageUrls);

        foreach ($localImages as $localImage) {
            $image = new Image();
            $image->setImageSrc($localImage->getImgSrc());
            $image->setBytes($localImage->getBytes());
            $image->setHeight($localImage->getHeight());
            $image->setWidth($localImage->getWidth());
            $image->setImageExtractionType('all');
            $image->setConfidenceScore(0);

            $results[] = $image;
        }

        return $results;
    }

    /**
     * returns true if we think this is kind of a bannery dimension
     * like 600 / 100 = 6 may be a fishy dimension for a good image
     *
     * @param int $width
     * @param int $height
     */
    private function isBannerDimensions($width, $height) {
        if ($width == $height) {
            return false;
        }

        if ($width > $height) {

            $diff = $width / $height;
            if ($diff > 5) {
                return true;
            }
        }

        if ($height > $width) {

            $diff = $height / $width;
            if ($diff > 5) {
                return true;
            }
        }

        return false;
    }

    /**
     * takes a list of image elements and filters out the ones with bad names
     *
     * @param \DOMWrap\NodeList $images
     *
     * @return Element[]
     */
    private function filterBadNames(NodeList $images) {
        $goodImages = [];

        foreach ($images as $image) {
            if ($this->isOkImageFileName($image)) {
                $goodImages[] = $image;
            } else {
                $image->remove();
            }
        }

        return $goodImages;
    }

    /**
     * will check the image src against a list of bad image files we know of like buttons, etc...
     *
     * @param Element $imageNode
     *
     * @return bool
     */
    private function isOkImageFileName(Element $imageNode) {
        $imgSrc = $imageNode->attr('src');

        if (empty($imgSrc)) {
            return false;
        }

        $regex = '@' . implode('|', $this->badFileNames) . '@i';

        if (preg_match($regex, $imgSrc)) {
            return false;
        }

        return true;
    }

    /**
     * @param Element $node
     *
     * @return LocallyStoredImage[]
     */
    private function getImageCandidates(Element $node) {
        $images = $node->find('img');
        $filteredImages = $this->filterBadNames($images);
        $goodImages = $this->findImagesThatPassByteSizeTest($filteredImages);

        return $goodImages;
    }

    /**
     * loop through all the images and find the ones that have the best bytes to even make them a candidate
     *
     * @param Element[] $images
     *
     * @return LocallyStoredImage[]
     */
    private function findImagesThatPassByteSizeTest($images) {
        $i = 0; /** @todo Re-factor how the LocallyStoredImage => Image relation works ? Note: PHP 5.6.x adds a 3rd argument to array_filter() to pass the key as well as value. */

        // Limit to the first 30 images
        $images = array_slice($images, 0, 30);

        // Generate a complete URL for each image
        $imageUrls = array_map(function($image) {
            return $this->buildImagePath($image->attr('src'));
        }, $images);

        $localImages = $this->getLocallyStoredImages($imageUrls, true);

        $results = array_filter($localImages, function($localImage) use($images, $i) {
            $image = $images[$i++];

            $bytes = $localImage->getBytes();

            if ($bytes < $this->config()->get('image_min_bytes') && $bytes != 0 || $bytes > $this->config()->get('image_max_bytes')) {
                $image->remove();

                return false;
            }

            return true;
        });

        return $results;
    }

    /**
     * checks to see if we were able to find feature image tags on this page
     *
     * @return Image|null
     */
    private function checkForLinkTag() {
        return $this->checkForTag('link[rel="image_src"]', 'href', 'linktag');
    }

    /**
     * checks to see if we were able to find open graph tags on this page
     *
     * @return Image|null
     */
    private function checkForOpenGraphTag() {
        return $this->checkForTag('meta[property="og:image"],meta[name="og:image"]', 'content', 'opengraph');
    }

    /**
     * checks to see if we were able to find twitter tags on this page
     *
     * @return Image|null
     */
    private function checkForTwitterTag() {
        return $this->checkForTag('meta[property="twitter:image"],meta[name="twitter:image"],meta[property="twitter:image:src"],meta[name="twitter:image:src"]', 'content', 'twitter');
    }

    /**
     * @param string $selector
     * @param string $attr
     * @param string $type
     *
     * @return Image|null
     */
    private function checkForTag($selector, $attr, $type) {
        $meta = $this->article()->getRawDoc()->find($selector);

        if (!$meta->count()) {
            return null;
        }

        $node = $meta->first();

        if (!($node instanceof Element)) {
            return null;
        }

        if (!$node->hasAttribute($attr)) {
            return null;
        }

        $imagePath = $this->buildImagePath($node->attr($attr));

        $mainImage = new Image();
        $mainImage->setImageSrc($imagePath);
        $mainImage->setImageExtractionType($type);
        $mainImage->setConfidenceScore(100);

        $locallyStoredImage = $this->getLocallyStoredImage($mainImage->getImageSrc());

        if (!empty($locallyStoredImage)) {

            $mainImage->setBytes($locallyStoredImage->getBytes());
            $mainImage->setHeight($locallyStoredImage->getHeight());
            $mainImage->setWidth($locallyStoredImage->getWidth());
        }

        return $this->ensureMinimumImageSize($mainImage);
    }

    /**
     * @param Image $mainImage
     *
     * @return Image|null
     */
    private function ensureMinimumImageSize(Image $mainImage) {
        if ($mainImage->getWidth() >= $this->config()->get('image_min_width')
          && $mainImage->getHeight() >= $this->config()->get('image_min_height')) {
            return $mainImage;
        }

        return null;
    }

    /**
     * @param string $imageSrc
     * @param bool $returnAll
     *
     * @return LocallyStoredImage|null
     */
    private function getLocallyStoredImage($imageSrc, $returnAll = false) {
        $locallyStoredImages = ImageUtils::storeImagesToLocalFile([$imageSrc], $returnAll, $this->config());

        return array_shift($locallyStoredImages);
    }

    /**
     * @param string[] $imageSrcs
     * @param bool $returnAll
     *
     * @return LocallyStoredImage[]
     */
    private function getLocallyStoredImages($imageSrcs, $returnAll = false) {
        return ImageUtils::storeImagesToLocalFile($imageSrcs, $returnAll, $this->config());
    }

    /**
     * @return string
     */
    private function getCleanDomain() {
        return implode('.', array_slice(explode('.', $this->article()->getDomain()), -2, 2));
    }

    /**
     * In here we check for known image contains from sites we've checked out like yahoo, techcrunch, etc... that have
     * known  places to look for good images.
     *
     * @todo enable this to use a series of settings files so people can define what the image ids/classes are on specific sites
     *
     * @return Image|null
     */
    private function checkForKnownElements() {
        if (!$this->article()->getRawDoc()) {
            return null;
        }

        $knownImgDomNames = self::$KNOWN_IMG_DOM_NAMES;

        $domain = $this->getCleanDomain();

        $customSiteMapping = $this->customSiteMapping();

        if (isset($customSiteMapping[$domain])) {
            foreach (explode('|', $customSiteMapping[$domain]) as $class) {
                $knownImgDomNames[] = $class;
            }
        }

        $knownImage = null;

        foreach ($knownImgDomNames as $knownName) {
            $known = $this->article()->getRawDoc()->find('#' . $knownName);

            if (!$known->count()) {
                $known = $this->article()->getRawDoc()->find('.' . $knownName);
            }

            if ($known->count()) {
                $mainImage = $known->first()->find('img');

                if ($mainImage->count()) {
                    $knownImage = $mainImage->first();
                }
            }
        }

        if (is_null($knownImage)) {
            return null;
        }

        $knownImgSrc = $knownImage->attr('src');

        $mainImage = new Image();
        $mainImage->setImageSrc($this->buildImagePath($knownImgSrc));
        $mainImage->setImageExtractionType('known');
        $mainImage->setConfidenceScore(90);

        $locallyStoredImage = $this->getLocallyStoredImage($mainImage->getImageSrc());

        if (!empty($locallyStoredImage)) {

            $mainImage->setBytes($locallyStoredImage->getBytes());
            $mainImage->setHeight($locallyStoredImage->getHeight());
            $mainImage->setWidth($locallyStoredImage->getWidth());
        }

        return $this->ensureMinimumImageSize($mainImage);
    }

    /**
     * This method will take an image path and build out the absolute path to that image
     * using the initial url we crawled so we can find a link to the image if they use relative urls like ../myimage.jpg
     *
     * @param string $imageSrc
     *
     * @return string
     */
    private function buildImagePath($imageSrc) {
        $parts = array(
            'scheme',
            'host',
            'port',
            'path',
            'query',
        );

        $imageUrlParts = parse_url($imageSrc);
        $articleUrlParts = parse_url($this->article()->getFinalUrl());
        if (isset($imageUrlParts['path'], $articleUrlParts['path']) && $imageUrlParts['path'] && $imageUrlParts['path']{0} !== '/') {
            $articleUrlDir = dirname($articleUrlParts['path']);
            $imageUrlParts['path'] = $articleUrlDir . '/' . $imageUrlParts['path'];
        }

        foreach ($parts as $part) {
            if (!isset($imageUrlParts[$part]) && isset($articleUrlParts[$part])) {
                $imageUrlParts[$part] = $articleUrlParts[$part];

            } else if (isset($imageUrlParts[$part]) && !isset($articleUrlParts[$part])) {
                break;
            }
        }

        return http_build_url($imageUrlParts, array());
    }

    /**
     * @param string[]
     */
    private function customSiteMapping() {
        if (empty(self::$CUSTOM_SITE_MAPPING)) {
            $file = __DIR__ . '/../../../resources/images/known-image-css.txt';

            $lines = explode("\n", str_replace(["\r\n", "\r"], "\n", file_get_contents($file)));

            foreach ($lines as $line) {
                list($domain, $css) = explode('^', $line);

                self::$CUSTOM_SITE_MAPPING[$domain] = $css;
            }
        }

        return self::$CUSTOM_SITE_MAPPING;
    }

}


1		<?php
2
3		namespace Goose\Modules\Extractors;
4
5		use Goose\Article;
6		use Goose\Images\Image;
7		use Goose\Images\ImageUtils;
8		use Goose\Images\LocallyStoredImage;
9		use Goose\Traits\ArticleMutatorTrait;
10		use Goose\Modules\AbstractModule;
11		use Goose\Modules\ModuleInterface;
12		use DOMWrap\Element;
13		use DOMWrap\NodeList;
14
15		/**
16		* Image Extractor
17		*
18		* @package Goose\Modules\Extractors
19		* @license http://www.apache.org/licenses/LICENSE-2.0 Apache License 2.0
20		*/
21		class ImageExtractor extends AbstractModule implements ModuleInterface {
22		use ArticleMutatorTrait;
23
24		/** @var string[] */
25		private $badFileNames = [
26		'\.html', '\.gif', '\.ico', 'button', 'twitter\.jpg', 'facebook\.jpg',
27		'ap_buy_photo', 'digg\.jpg', 'digg\.png', 'delicious\.png',
28		'facebook\.png', 'reddit\.jpg', 'doubleclick', 'diggthis',
29		'diggThis', 'adserver', '\/ads\/', 'ec\.atdmt\.com', 'mediaplex\.com',
30		'adsatt', 'view\.atdmt',
31		];
32
33		/** @var string[] */
34		private static $KNOWN_IMG_DOM_NAMES = [
35		'yn-story-related-media',
36		'cnn_strylccimg300cntr',
37		'big_photo',
38		'ap-smallphoto-a'
39		];
40
41		/** @var int */
42		private static $MAX_PARENT_DEPTH = 2;
43
44		/** @var string[] */
45		private static $CUSTOM_SITE_MAPPING = [];
46
47		/**
48		* @param Article $article
49		*/
50		public function run(Article $article) {
51		$this->article($article);
52
53		if ($this->config()->get('image_fetch_best')) {
54		$article->setTopImage($this->getBestImage());
55
56		if ($this->config()->get('image_fetch_all')
57		&& $article->getTopNode() instanceof Element) {
58		$article->setAllImages($this->getAllImages());
59		}
60		}
61		}
62
63		/**
64		* @return Image\|null
65		*/
66		private function getBestImage() {
67		$image = $this->checkForKnownElements();
68
69		if ($image) {
70		return $image;
71		}
72
73		$image = $this->checkForMetaTag();
74
75		if ($image) {
76		return $image;
77		}
78
79		if ($this->article()->getTopNode() instanceof Element) {
80		$image = $this->checkForLargeImages($this->article()->getTopNode(), 0, 0);
81
82		if ($image) {
83		return $image;
84		}
85		}
86
87		return null;
88		}
89
90		/**
91		* Prefer Twitter images (as they tend to have the right size for us), then Open Graph images
92		* (which seem to be smaller), and finally linked images.
93		*
94		* @return Image\|null
95		*/
96		private function checkForMetaTag() {
97		$image = $this->checkForTwitterTag();
98
99		if ($image) {
100		return $image;
101		}
102
103		$image = $this->checkForOpenGraphTag();
104
105		if ($image) {
106		return $image;
107		}
108
109		$image = $this->checkForLinkTag();
110
111		if ($image) {
112		return $image;
113		}
114
115		return null;
116		}
117
118		/**
119		* although slow the best way to determine the best image is to download them and check the actual dimensions of the image when on disk
120		* so we'll go through a phased approach...
121		* 1. get a list of ALL images from the parent node
122		* 2. filter out any bad image names that we know of (gifs, ads, etc..)
123		* 3. do a head request on each file to make sure it meets our bare requirements
124		* 4. any images left over let's do a full GET request, download em to disk and check their dimensions
125		* 5. Score images based on different factors like height/width and possibly things like color density
126		*
127		* @param Element $node
128		* @param int $parentDepthLevel
129		* @param int $siblingDepthLevel
130		*
131		* @return Image\|null
132		*/
133		private function checkForLargeImages(Element $node, $parentDepthLevel, $siblingDepthLevel) {
134		$goodLocalImages = $this->getImageCandidates($node);
135
136		$scoredLocalImages = $this->scoreLocalImages($goodLocalImages, $parentDepthLevel);
137
138		ksort($scoredLocalImages);
139
140		if (!empty($scoredLocalImages)) {
141		foreach ($scoredLocalImages as $imageScore => $scoredLocalImage) {
142		$mainImage = new Image();
143		$mainImage->setImageSrc($scoredLocalImage->getImgSrc());
144		$mainImage->setImageExtractionType('bigimage');
145		$mainImage->setConfidenceScore(100 / count($scoredLocalImages));
146		$mainImage->setImageScore($imageScore);
147		$mainImage->setBytes($scoredLocalImage->getBytes());
148		$mainImage->setHeight($scoredLocalImage->getHeight());
149		$mainImage->setWidth($scoredLocalImage->getWidth());
150
151		return $mainImage;
152		}
153		} else {
154		$depthObj = $this->getDepthLevel($node, $parentDepthLevel, $siblingDepthLevel);
155
156		if ($depthObj && NULL !== $depthObj->node) {
157		return $this->checkForLargeImages($depthObj->node, $depthObj->parentDepth, $depthObj->siblingDepth);
158		}
159		}
160
161		return null;
162		}
163
164		/**
165		* @param Element $node
166		* @param int $parentDepth
167		* @param int $siblingDepth
168		*
169		* @return object\|null
170		*/
171		private function getDepthLevel(Element $node, $parentDepth, $siblingDepth) {
172		if (is_null($node) \|\| !($node->parent() instanceof Element)) {
173		return null;
174		}
175
176		if ($parentDepth > self::$MAX_PARENT_DEPTH) {
177		return null;
178		}
179
180		// Find previous sibling element node
181		$siblingNode = $node->preceding(function($node) {
182		return $node instanceof Element;
183		});
184
185		if (is_null($siblingNode)) {
186		return (object)[
187		'node' => $node->parent(),
188		'parentDepth' => $parentDepth + 1,
189		'siblingDepth' => 0,
190		];
191		}
192
193		return (object)[
194		'node' => $siblingNode,
195		'parentDepth' => $parentDepth,
196		'siblingDepth' => $siblingDepth + 1,
197		];
198		}
199
200		/**
201		* Set image score and on locally downloaded images
202		*
203		* we're going to score the images in the order in which they appear so images higher up will have more importance,
204		* we'll count the area of the 1st image as a score of 1 and then calculate how much larger or small each image after it is
205		* we'll also make sure to try and weed out banner type ad blocks that have big widths and small heights or vice versa
206		* so if the image is 3rd found in the dom it's sequence score would be 1 / 3 = .33 * diff in area from the first image
207		*
208		* @param LocallyStoredImage[] $locallyStoredImages
209		* @param int $depthLevel
210		*
211		* @return LocallyStoredImage[]
212		*/
213		private function scoreLocalImages($locallyStoredImages, $depthLevel) {
		0 ignored issues – show Unused Code introduced 2017-08-02 12:12 UTC by Report Bug Copy Issue Report The parameter `$depthLevel` is not used and could be removed. This check looks from parameters that have been defined for a function or method, but which are not used in the method body. Loading history...
214		$results = [];
215		$i = 1;
216		$initialArea = 0;
217
218		// Limit to the first 30 images
219		$locallyStoredImages = array_slice($locallyStoredImages, 0, 30);
220
221		foreach ($locallyStoredImages as $locallyStoredImage) {
222		$sequenceScore = 1 / $i;
223		$area = $locallyStoredImage->getWidth() * $locallyStoredImage->getHeight();
224
225		if ($initialArea == 0) {
226		$initialArea = $area * 1.48;
227		$totalScore = 1;
228		} else {
229		$areaDifference = $area * $initialArea;
230		$totalScore = $sequenceScore * $areaDifference;
231		}
232
233		$i++;
234
235		$results[$totalScore] = $locallyStoredImage;
236		}
237
238		return $results;
239		}
240
241		/**
242		* @param LocallyStoredImage $locallyStoredImage
243		* @param int $depthLevel
244		*
245		* @return bool
246		*/
247		private function isWorthyImage($locallyStoredImage, $depthLevel) {
		0 ignored issues – show Unused Code introduced 2017-08-02 12:12 UTC by Report Bug Copy Issue Report This method is not used, and could be removed. Loading history...
248		if ($locallyStoredImage->getWidth() <= $this->config()->get('image_min_width')
249		\|\| $locallyStoredImage->getHeight() <= $this->config()->get('image_min_height')
250		\|\| $locallyStoredImage->getFileExtension() == 'NA'
251		\|\| ($depthLevel < 1 && $locallyStoredImage->getWidth() < 300) \|\| $depthLevel >= 1
252		\|\| $this->isBannerDimensions($locallyStoredImage->getWidth(), $locallyStoredImage->getHeight())) {
253		return false;
254		}
255
256		return true;
257		}
258
259		/**
260		* @return Image[]
261		*/
262		private function getAllImages() {
263		$results = [];
264
265		$images = $this->article()->getTopNode()->find('img');
266
267		// Generate a complete URL for each image
268		$imageUrls = array_map(function($image) {
269		return $this->buildImagePath($image->attr('src'));
270		}, $images->toArray());
271
272		$localImages = $this->getLocallyStoredImages($imageUrls);
273
274		foreach ($localImages as $localImage) {
275		$image = new Image();
276		$image->setImageSrc($localImage->getImgSrc());
277		$image->setBytes($localImage->getBytes());
278		$image->setHeight($localImage->getHeight());
279		$image->setWidth($localImage->getWidth());
280		$image->setImageExtractionType('all');
281		$image->setConfidenceScore(0);
282
283		$results[] = $image;
284		}
285
286		return $results;
287		}
288
289		/**
290		* returns true if we think this is kind of a bannery dimension
291		* like 600 / 100 = 6 may be a fishy dimension for a good image
292		*
293		* @param int $width
294		* @param int $height
295		*/
296		private function isBannerDimensions($width, $height) {
297		if ($width == $height) {
298		return false;
299		}
300
301	View Code Duplication	if ($width > $height) {
		0 ignored issues – show Duplication introduced 2016-04-13 18:21 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
302		$diff = $width / $height;
303		if ($diff > 5) {
304		return true;
305		}
306		}
307
308	View Code Duplication	if ($height > $width) {
		0 ignored issues – show Duplication introduced 2016-04-13 18:21 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
309		$diff = $height / $width;
310		if ($diff > 5) {
311		return true;
312		}
313		}
314
315		return false;
316		}
317
318		/**
319		* takes a list of image elements and filters out the ones with bad names
320		*
321		* @param \DOMWrap\NodeList $images
322		*
323		* @return Element[]
324		*/
325		private function filterBadNames(NodeList $images) {
326		$goodImages = [];
327
328		foreach ($images as $image) {
329		if ($this->isOkImageFileName($image)) {
330		$goodImages[] = $image;
331		} else {
332		$image->remove();
333		}
334		}
335
336		return $goodImages;
337		}
338
339		/**
340		* will check the image src against a list of bad image files we know of like buttons, etc...
341		*
342		* @param Element $imageNode
343		*
344		* @return bool
345		*/
346		private function isOkImageFileName(Element $imageNode) {
347		$imgSrc = $imageNode->attr('src');
348
349		if (empty($imgSrc)) {
350		return false;
351		}
352
353		$regex = '@' . implode('\|', $this->badFileNames) . '@i';
354
355		if (preg_match($regex, $imgSrc)) {
356		return false;
357		}
358
359		return true;
360		}
361
362		/**
363		* @param Element $node
364		*
365		* @return LocallyStoredImage[]
366		*/
367		private function getImageCandidates(Element $node) {
368		$images = $node->find('img');
369		$filteredImages = $this->filterBadNames($images);
370		$goodImages = $this->findImagesThatPassByteSizeTest($filteredImages);
371
372		return $goodImages;
373		}
374
375		/**
376		* loop through all the images and find the ones that have the best bytes to even make them a candidate
377		*
378		* @param Element[] $images
379		*
380		* @return LocallyStoredImage[]
381		*/
382		private function findImagesThatPassByteSizeTest($images) {
383		$i = 0; /** @todo Re-factor how the LocallyStoredImage => Image relation works ? Note: PHP 5.6.x adds a 3rd argument to array_filter() to pass the key as well as value. */
384
385		// Limit to the first 30 images
386		$images = array_slice($images, 0, 30);
387
388		// Generate a complete URL for each image
389		$imageUrls = array_map(function($image) {
390		return $this->buildImagePath($image->attr('src'));
391		}, $images);
392
393		$localImages = $this->getLocallyStoredImages($imageUrls, true);
394
395		$results = array_filter($localImages, function($localImage) use($images, $i) {
396		$image = $images[$i++];
397
398		$bytes = $localImage->getBytes();
399
400		if ($bytes < $this->config()->get('image_min_bytes') && $bytes != 0 \|\| $bytes > $this->config()->get('image_max_bytes')) {
401		$image->remove();
402
403		return false;
404		}
405
406		return true;
407		});
408
409		return $results;
410		}
411
412		/**
413		* checks to see if we were able to find feature image tags on this page
414		*
415		* @return Image\|null
416		*/
417		private function checkForLinkTag() {
418		return $this->checkForTag('link[rel="image_src"]', 'href', 'linktag');
419		}
420
421		/**
422		* checks to see if we were able to find open graph tags on this page
423		*
424		* @return Image\|null
425		*/
426		private function checkForOpenGraphTag() {
427		return $this->checkForTag('meta[property="og:image"],meta[name="og:image"]', 'content', 'opengraph');
428		}
429
430		/**
431		* checks to see if we were able to find twitter tags on this page
432		*
433		* @return Image\|null
434		*/
435		private function checkForTwitterTag() {
436		return $this->checkForTag('meta[property="twitter:image"],meta[name="twitter:image"],meta[property="twitter:image:src"],meta[name="twitter:image:src"]', 'content', 'twitter');
437		}
438
439		/**
440		* @param string $selector
441		* @param string $attr
442		* @param string $type
443		*
444		* @return Image\|null
445		*/
446		private function checkForTag($selector, $attr, $type) {
447		$meta = $this->article()->getRawDoc()->find($selector);
448
449		if (!$meta->count()) {
450		return null;
451		}
452
453		$node = $meta->first();
454
455		if (!($node instanceof Element)) {
456		return null;
457		}
458
459		if (!$node->hasAttribute($attr)) {
460		return null;
461		}
462
463		$imagePath = $this->buildImagePath($node->attr($attr));
		0 ignored issues – show Bug introduced 2016-04-13 18:21 UTC by Report Bug Copy Issue Report It seems like `$node->attr($attr)` targeting `DOMWrap\Traits\ManipulationTrait::attr()` can also be of type `null` or `object<DOMWrap\Element>`; however, `Goose\Modules\Extractors...actor::buildImagePath()` does only seem to accept `string`, maybe add an additional type check? This check looks at variables that are passed out again to other methods. If the outgoing method call has stricter type requirements than the method itself, an issue is raised. An additional type check may prevent trouble. Loading history...
464		$mainImage = new Image();
465		$mainImage->setImageSrc($imagePath);
466		$mainImage->setImageExtractionType($type);
467		$mainImage->setConfidenceScore(100);
468
469		$locallyStoredImage = $this->getLocallyStoredImage($mainImage->getImageSrc());
470
471	View Code Duplication	if (!empty($locallyStoredImage)) {
		0 ignored issues – show Duplication introduced 2016-04-13 18:21 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
472		$mainImage->setBytes($locallyStoredImage->getBytes());
473		$mainImage->setHeight($locallyStoredImage->getHeight());
474		$mainImage->setWidth($locallyStoredImage->getWidth());
475		}
476
477		return $this->ensureMinimumImageSize($mainImage);
478		}
479
480		/**
481		* @param Image $mainImage
482		*
483		* @return Image\|null
484		*/
485		private function ensureMinimumImageSize(Image $mainImage) {
486		if ($mainImage->getWidth() >= $this->config()->get('image_min_width')
487		&& $mainImage->getHeight() >= $this->config()->get('image_min_height')) {
488		return $mainImage;
489		}
490
491		return null;
492		}
493
494		/**
495		* @param string $imageSrc
496		* @param bool $returnAll
497		*
498		* @return LocallyStoredImage\|null
499		*/
500		private function getLocallyStoredImage($imageSrc, $returnAll = false) {
501		$locallyStoredImages = ImageUtils::storeImagesToLocalFile([$imageSrc], $returnAll, $this->config());
502
503		return array_shift($locallyStoredImages);
504		}
505
506		/**
507		* @param string[] $imageSrcs
508		* @param bool $returnAll
509		*
510		* @return LocallyStoredImage[]
511		*/
512		private function getLocallyStoredImages($imageSrcs, $returnAll = false) {
513		return ImageUtils::storeImagesToLocalFile($imageSrcs, $returnAll, $this->config());
514		}
515
516		/**
517		* @return string
518		*/
519		private function getCleanDomain() {
520		return implode('.', array_slice(explode('.', $this->article()->getDomain()), -2, 2));
521		}
522
523		/**
524		* In here we check for known image contains from sites we've checked out like yahoo, techcrunch, etc... that have
525		* known places to look for good images.
526		*
527		* @todo enable this to use a series of settings files so people can define what the image ids/classes are on specific sites
528		*
529		* @return Image\|null
530		*/
531		private function checkForKnownElements() {
532		if (!$this->article()->getRawDoc()) {
533		return null;
534		}
535
536		$knownImgDomNames = self::$KNOWN_IMG_DOM_NAMES;
537
538		$domain = $this->getCleanDomain();
539
540		$customSiteMapping = $this->customSiteMapping();
541
542		if (isset($customSiteMapping[$domain])) {
543		foreach (explode('\|', $customSiteMapping[$domain]) as $class) {
544		$knownImgDomNames[] = $class;
545		}
546		}
547
548		$knownImage = null;
549
550		foreach ($knownImgDomNames as $knownName) {
551		$known = $this->article()->getRawDoc()->find('#' . $knownName);
552
553		if (!$known->count()) {
554		$known = $this->article()->getRawDoc()->find('.' . $knownName);
555		}
556
557		if ($known->count()) {
558		$mainImage = $known->first()->find('img');
559
560		if ($mainImage->count()) {
561		$knownImage = $mainImage->first();
562		}
563		}
564		}
565
566		if (is_null($knownImage)) {
567		return null;
568		}
569
570		$knownImgSrc = $knownImage->attr('src');
571
572		$mainImage = new Image();
573		$mainImage->setImageSrc($this->buildImagePath($knownImgSrc));
574		$mainImage->setImageExtractionType('known');
575		$mainImage->setConfidenceScore(90);
576
577		$locallyStoredImage = $this->getLocallyStoredImage($mainImage->getImageSrc());
578
579	View Code Duplication	if (!empty($locallyStoredImage)) {
		0 ignored issues – show Duplication introduced 2016-04-13 18:21 UTC by Report Bug Copy Issue Report This code seems to be duplicated across your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
580		$mainImage->setBytes($locallyStoredImage->getBytes());
581		$mainImage->setHeight($locallyStoredImage->getHeight());
582		$mainImage->setWidth($locallyStoredImage->getWidth());
583		}
584
585		return $this->ensureMinimumImageSize($mainImage);
586		}
587
588		/**
589		* This method will take an image path and build out the absolute path to that image
590		* using the initial url we crawled so we can find a link to the image if they use relative urls like ../myimage.jpg
591		*
592		* @param string $imageSrc
593		*
594		* @return string
595		*/
596		private function buildImagePath($imageSrc) {
597		$parts = array(
598		'scheme',
599		'host',
600		'port',
601		'path',
602		'query',
603		);
604
605		$imageUrlParts = parse_url($imageSrc);
606		$articleUrlParts = parse_url($this->article()->getFinalUrl());
607		if (isset($imageUrlParts['path'], $articleUrlParts['path']) && $imageUrlParts['path'] && $imageUrlParts['path']{0} !== '/') {
608		$articleUrlDir = dirname($articleUrlParts['path']);
609		$imageUrlParts['path'] = $articleUrlDir . '/' . $imageUrlParts['path'];
610		}
611
612		foreach ($parts as $part) {
613		if (!isset($imageUrlParts[$part]) && isset($articleUrlParts[$part])) {
614		$imageUrlParts[$part] = $articleUrlParts[$part];
615
616		} else if (isset($imageUrlParts[$part]) && !isset($articleUrlParts[$part])) {
617		break;
618		}
619		}
620
621		return http_build_url($imageUrlParts, array());
622		}
623
624		/**
625		* @param string[]
626		*/
627		private function customSiteMapping() {
628		if (empty(self::$CUSTOM_SITE_MAPPING)) {
629		$file = __DIR__ . '/../../../resources/images/known-image-css.txt';
630
631		$lines = explode("\n", str_replace(["\r\n", "\r"], "\n", file_get_contents($file)));
632
633		foreach ($lines as $line) {
634		list($domain, $css) = explode('^', $line);
635
636		self::$CUSTOM_SITE_MAPPING[$domain] = $css;
637		}
638		}
639
640		return self::$CUSTOM_SITE_MAPPING;
641		}
642
643		}
644

scotteh / php-goose

Push — master ( 84b1b9...90721d )

ImageExtractor::buildImagePath() D

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like