Crawler::getGoogleUrl() - Code Metrics - Inspection of "composer.lock updated" - CViniciusSDias/google-crawler - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 741aff...f3b872 )

by Vinicius

created 2017-07-29 23:51 UTC

Crawler::getGoogleUrl() A

↳ Parent: Crawler

Complexity

Conditions	2
Paths	2

Size

Total Lines	10
Code Lines	6

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	6
CRAP Score	2

Importance

Changes

Metric	Value
dl	0
loc	10
ccs	6
cts	6
cp	1
rs	9.4285
c	0
b	0
f	0
cc	2
eloc	6
nc	2
nop	0
crap	2

<?php
namespace CViniciusSDias\GoogleCrawler;

use CViniciusSDias\GoogleCrawler\Exception\InvalidResultException;
use CViniciusSDias\GoogleCrawler\Proxy\{
    GoogleProxyInterface, NoProxy
};
use Psr\Http\Message\ResponseInterface;
use Symfony\Component\DomCrawler\Crawler as DomCrawler;
use Symfony\Component\DomCrawler\Link;
use DOMElement;

/**
 * Google Crawler
 *
 * @package CViniciusSDias\GoogleCrawler
 * @author Vinicius Dias
 */
class Crawler
{
    /** @var GoogleProxyInterface $proxy */
    protected $proxy;
    /** @var SearchTermInterface $searchTerm */
    private $searchTerm;
    /** @var string $countrySpecificSuffix */
    private $googleDomain;
    /** @var string $countryCode */
    private $countryCode;

    public function __construct(
        SearchTermInterface $searchTerm, GoogleProxyInterface $proxy = null,
        string $googleDomain = 'google.com', string $countryCode = ''
    ) {
        $this->proxy = is_null($proxy) ? new NoProxy() : $proxy;

        $this->searchTerm = $searchTerm;

        if (mb_stripos($googleDomain, 'google.') === false || mb_stripos($googleDomain, 'http') === 0) {
            throw new \InvalidArgumentException('Invalid google domain');
        }
        $this->googleDomain = $googleDomain;

        $this->countryCode = mb_strtoupper($countryCode);
    }

    /**
     * Returns the 100 first found results for the specified search term
     *
     * @return ResultList
     * @throws \GuzzleHttp\Exception\ServerException If the proxy was overused
     * @throws \GuzzleHttp\Exception\ConnectException If the proxy is unavailable or $countrySpecificSuffix is invalid
     */
    public function getResults(): ResultList
    {
        $googleUrl = $this->getGoogleUrl();
        /** @var ResponseInterface $response */
        $response = $this->proxy->getHttpResponse($googleUrl);
        $stringResponse = (string) $response->getBody();
        $domCrawler = new DomCrawler($stringResponse);
        $googleResults = $domCrawler->filterXPath('//div[@class="g" and h3[@class="r" and a]]');
        $resultList = new ResultList($googleResults->count());

        foreach ($googleResults as $result) {
            $resultCrawler = new DomCrawler($result);
            $linkElement = $resultCrawler->filterXPath('//h3[@class="r"]/a')->getNode(0);
            $resultLink = new Link($linkElement, 'http://google.com/');
/** @return stdClass|null */
function mayReturnNull() { }

function doesNotAcceptNull(stdClass $x) { }

// With potential error.
function withoutCheck() {
    $x = mayReturnNull();
    doesNotAcceptNull($x); // Potential error here.
}

// Safe - Alternative 1
function withCheck1() {
    $x = mayReturnNull();
    if ( ! $x instanceof stdClass) {
        throw new \LogicException('$x must be defined.');
    }
    doesNotAcceptNull($x);
}

// Safe - Alternative 2
function withCheck2() {
    $x = mayReturnNull();
    if ($x instanceof stdClass) {
        doesNotAcceptNull($x);
    }
}
            $descriptionElement = $resultCrawler->filterXPath('//span[@class="st"]')->getNode(0);
            try {
                $googleResult = $this->parseResult($resultLink, $descriptionElement);
/** @return stdClass|null */
function mayReturnNull() { }

function doesNotAcceptNull(stdClass $x) { }

// With potential error.
function withoutCheck() {
    $x = mayReturnNull();
    doesNotAcceptNull($x); // Potential error here.
}

// Safe - Alternative 1
function withCheck1() {
    $x = mayReturnNull();
    if ( ! $x instanceof stdClass) {
        throw new \LogicException('$x must be defined.');
    }
    doesNotAcceptNull($x);
}

// Safe - Alternative 2
function withCheck2() {
    $x = mayReturnNull();
    if ($x instanceof stdClass) {
        doesNotAcceptNull($x);
    }
}
                $resultList->addResult($googleResult);
            } catch (InvalidResultException $invalidResult) {
                // Maybe log this exception. Other than that, there's nothing to do, cause it isn't an error.
            }
        }

        return $resultList;
    }

    /**
     * If $resultLink is a valid link, this method assembles the Result and adds it to $googleResults
     *
     * @param Link $resultLink
     * @param DOMElement $descriptionElement
     * @return Result
     * @throws InvalidResultException
     */
    private function parseResult(Link $resultLink, DOMElement $descriptionElement): Result
    {
        $description = $descriptionElement->nodeValue
            ?? 'A description for this result isn\'t available due to the robots.txt file.';

        $googleResult = new Result();
        $googleResult
            ->setTitle($resultLink->getNode()->nodeValue)
            ->setUrl($this->parseUrl($resultLink->getUri()))
            ->setDescription($description);

        return $googleResult;
    }

    /**
     * Parses the URL using the parser provided by $proxy
     *
     * @param string $url
     * @return string
     * @throws InvalidResultException
     */
    private function parseUrl(string $url): string
    {
        return $this->proxy->parseUrl($url);
    }

    /**
     * Assembles the Google URL using the previously informed data
     */
    private function getGoogleUrl(): string
    {
        $domain = $this->googleDomain;
        $url = "https://$domain/search?q={$this->searchTerm}&num=100";
        if (!empty($this->countryCode)) {
            $url .= "&gl={$this->countryCode}";
        }

        return $url;
    }
}


1		<?php
2		namespace CViniciusSDias\GoogleCrawler;
3
4		use CViniciusSDias\GoogleCrawler\Exception\InvalidResultException;
5		use CViniciusSDias\GoogleCrawler\Proxy\{
6		GoogleProxyInterface, NoProxy
7		};
8		use Psr\Http\Message\ResponseInterface;
9		use Symfony\Component\DomCrawler\Crawler as DomCrawler;
10		use Symfony\Component\DomCrawler\Link;
11		use DOMElement;
12
13		/**
14		* Google Crawler
15		*
16		* @package CViniciusSDias\GoogleCrawler
17		* @author Vinicius Dias
18		*/
19		class Crawler
20		{
21		/** @var GoogleProxyInterface $proxy */
22		protected $proxy;
23		/** @var SearchTermInterface $searchTerm */
24		private $searchTerm;
25		/** @var string $countrySpecificSuffix */
26		private $googleDomain;
27		/** @var string $countryCode */
28		private $countryCode;
29
30	19	public function __construct(
31		SearchTermInterface $searchTerm, GoogleProxyInterface $proxy = null,
32		string $googleDomain = 'google.com', string $countryCode = ''
33		) {
34	19	$this->proxy = is_null($proxy) ? new NoProxy() : $proxy;
		0 ignored issues – show Documentation Bug introduced 2017-07-29 23:50 UTC by Report Bug Copy Issue Report It seems like `is_null($proxy) ? new \C...roxy\NoProxy() : $proxy` of type `object<CViniciusSDias\Go...y\GoogleProxyInterface>` is incompatible with the declared type `object<GoogleProxyInterface>` of property `$proxy`. Our type inference engine has found an assignment to a property that is incompatible with the declared type of that property. Either this assignment is in error or the assigned type should be added to the documentation/type hint for that property.. Loading history...
35	19	$this->searchTerm = $searchTerm;
36
37	19	if (mb_stripos($googleDomain, 'google.') === false \|\| mb_stripos($googleDomain, 'http') === 0) {
38	2	throw new \InvalidArgumentException('Invalid google domain');
39		}
40	17	$this->googleDomain = $googleDomain;
41
42	17	$this->countryCode = mb_strtoupper($countryCode);
43	17	}
44
45		/**
46		* Returns the 100 first found results for the specified search term
47		*
48		* @return ResultList
49		* @throws \GuzzleHttp\Exception\ServerException If the proxy was overused
50		* @throws \GuzzleHttp\Exception\ConnectException If the proxy is unavailable or $countrySpecificSuffix is invalid
51		*/
52	14	public function getResults(): ResultList
53		{
54	14	$googleUrl = $this->getGoogleUrl();
55		/** @var ResponseInterface $response */
56	14	$response = $this->proxy->getHttpResponse($googleUrl);
57	13	$stringResponse = (string) $response->getBody();
58	13	$domCrawler = new DomCrawler($stringResponse);
59	13	$googleResults = $domCrawler->filterXPath('//div[@class="g" and h3[@class="r" and a]]');
60	13	$resultList = new ResultList($googleResults->count());
61
62	13	foreach ($googleResults as $result) {
63	13	$resultCrawler = new DomCrawler($result);
64	13	$linkElement = $resultCrawler->filterXPath('//h3[@class="r"]/a')->getNode(0);
65	13	$resultLink = new Link($linkElement, 'http://google.com/');
		0 ignored issues – show Bug introduced 2017-07-02 18:03 UTC by Report Bug Copy Issue Report It seems like `$linkElement` defined by `$resultCrawler->filterXP...ss="r"]/a')->getNode(0)` on line `64` can be `null`; however, `Symfony\Component\DomCra...iElement::__construct()` does not accept `null`, maybe add an additional type check? Unless you are absolutely sure that the expression can never be null because of other conditions, we strongly recommend to add an additional type check to your code: /** @return stdClass\|null */ function mayReturnNull() { } function doesNotAcceptNull(stdClass $x) { } // With potential error. function withoutCheck() { $x = mayReturnNull(); doesNotAcceptNull($x); // Potential error here. } // Safe - Alternative 1 function withCheck1() { $x = mayReturnNull(); if ( ! $x instanceof stdClass) { throw new \LogicException('$x must be defined.'); } doesNotAcceptNull($x); } // Safe - Alternative 2 function withCheck2() { $x = mayReturnNull(); if ($x instanceof stdClass) { doesNotAcceptNull($x); } } Loading history...
66	13	$descriptionElement = $resultCrawler->filterXPath('//span[@class="st"]')->getNode(0);
67		try {
68	13	$googleResult = $this->parseResult($resultLink, $descriptionElement);
		0 ignored issues – show Bug introduced 2017-07-02 18:03 UTC by Report Bug Copy Issue Report It seems like `$descriptionElement` defined by `$resultCrawler->filterXP...ass="st"]')->getNode(0)` on line `66` can be `null`; however, `CViniciusSDias\GoogleCra...\Crawler::parseResult()` does not accept `null`, maybe add an additional type check? Unless you are absolutely sure that the expression can never be null because of other conditions, we strongly recommend to add an additional type check to your code: /** @return stdClass\|null */ function mayReturnNull() { } function doesNotAcceptNull(stdClass $x) { } // With potential error. function withoutCheck() { $x = mayReturnNull(); doesNotAcceptNull($x); // Potential error here. } // Safe - Alternative 1 function withCheck1() { $x = mayReturnNull(); if ( ! $x instanceof stdClass) { throw new \LogicException('$x must be defined.'); } doesNotAcceptNull($x); } // Safe - Alternative 2 function withCheck2() { $x = mayReturnNull(); if ($x instanceof stdClass) { doesNotAcceptNull($x); } } Loading history...
69	13	$resultList->addResult($googleResult);
70	13	} catch (InvalidResultException $invalidResult) {
71		// Maybe log this exception. Other than that, there's nothing to do, cause it isn't an error.
72		}
73		}
74
75	13	return $resultList;
76		}
77
78		/**
79		* If $resultLink is a valid link, this method assembles the Result and adds it to $googleResults
80		*
81		* @param Link $resultLink
82		* @param DOMElement $descriptionElement
83		* @return Result
84		* @throws InvalidResultException
85		*/
86	13	private function parseResult(Link $resultLink, DOMElement $descriptionElement): Result
87		{
88	13	$description = $descriptionElement->nodeValue
89	13	?? 'A description for this result isn\'t available due to the robots.txt file.';
90
91	13	$googleResult = new Result();
92		$googleResult
93	13	->setTitle($resultLink->getNode()->nodeValue)
94	13	->setUrl($this->parseUrl($resultLink->getUri()))
95	13	->setDescription($description);
96
97	13	return $googleResult;
98		}
99
100		/**
101		* Parses the URL using the parser provided by $proxy
102		*
103		* @param string $url
104		* @return string
105		* @throws InvalidResultException
106		*/
107	13	private function parseUrl(string $url): string
108		{
109	13	return $this->proxy->parseUrl($url);
110		}
111
112		/**
113		* Assembles the Google URL using the previously informed data
114		*/
115	17	private function getGoogleUrl(): string
116		{
117	17	$domain = $this->googleDomain;
118	17	$url = "https://$domain/search?q={$this->searchTerm}&num=100";
119	17	if (!empty($this->countryCode)) {
120	2	$url .= "&gl={$this->countryCode}";
121		}
122
123	17	return $url;
124		}
125		}
126

CViniciusSDias / google-crawler

Push — master ( 741aff...f3b872 )

Crawler::getGoogleUrl() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like