Harvest::getBaseUrl() - Code Metrics - Inspection of "fix bug" - PiedWeb/UrlHarvester - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( db32c8...b7e19f )

by Dev

created 2019-01-15 20:19 UTC

Harvest::getBaseUrl() A

↳ Parent: Harvest

Complexity

Conditions	5
Paths	3

Size

Total Lines	12
Code Lines	7

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	6
CRAP Score	5.0729

Importance

Changes

Metric	Value
cc	5
eloc	7
nc	3
nop	0
dl	0
loc	12
ccs	6
cts	7
cp	0.8571
crap	5.0729
rs	9.6111
c	0
b	0
f	0

<?php

namespace PiedWeb\UrlHarvester;

use PiedWeb\Curl\Response;
use PiedWeb\TextAnalyzer\Analyzer as TextAnalyzer;
use phpuri;
use simple_html_dom;
use Spatie\Robots\RobotsTxt;
use PiedWeb\Curl\Request as CurlRequest;

class Harvest
{
    use HarvestLinksTrait;

    const LINK_SELF = 1;
    const LINK_INTERNAL = 2;
    const LINK_SUB = 3;
    const LINK_EXTERNAL = 4;

    /**
     * @var Response
     */
    protected $response;

    /**
     * @var simple_html_dom
     */
    protected $dom;

    /** @var string */
    protected $baseUrl;

    /** @var string */
    protected $domain;

    /** @var RobotsTxt|string (empty string) */
    protected $robotsTxt;

    /** @var string */
    private $domainWithScheme;

    public static function fromUrl(
        string $url,
        string $userAgent = 'Bot: Url Harvester',
        string $language = 'en,en-US;q=0.5'
    ) {
        $response = Request::make($url, $userAgent, '200;html', $language);

        if ($response instanceof Response) {
            return new self($response);
        }

        return $response;
    }

    /**
     * @param Response $response
     */
    public function __construct(Response $response)
    {
        $this->response = $response;
    }

    public function getResponse()
    {
        return $this->response;
    }

    public function getRedirection()
    {
        $headers = $this->response->getHeaders();
        $headers = array_change_key_case($headers ? $headers : []);
        if (isset($headers['location'])) {
            return phpUri::parse($this->response->getEffectiveUrl())->join($headers['location']);
        }

        return false;
    }

    public function getDom()
    {
        if (null === $this->dom) {
            $this->dom = new simple_html_dom();
            $this->dom->load($this->response->getContent());
        }

        return $this->dom;
    }

    private function find($selector, $number = null)
    {
        return $this->getDom()->find($selector, $number);
    }

    private function findOne($selector)
    {
        return $this->find($selector, 0);
    }

    /**
     * Return content inside a selector.
     *
     * @return string
     */
    public function getTag($selector)
    {
        $found = $this->findOne($selector);

        return null !== $found ? Helper::clean($found->innertext) : null;
    }

    public function getUniqueTag($selector = 'title')
    {
        $found = $this->find($selector);
        if ($found) {
            if (count($found) > 1) {
                return count($found).' `'.$selector.'` !!';
            } else {
                return Helper::clean($found[0]->innertext);
            }
        }
    }

    /**
     * Return content inside a meta.
     *
     * @return string from content attribute
     */
    public function getMeta(string $name)
    {
        $meta = $this->findOne('meta[name='.$name.']');

        return null !== $meta ? (isset($meta->content) ? Helper::clean($meta->content) : '') : '';
    }

    /**
     * Renvoie le contenu de l'attribut href de la balise link rel=canonical.
     *
     * @return string le contenu de l'attribute href sinon NULL si la balise n'existe pas
     */
    public function getCanonical()
    {
        $canonical = $this->findOne('link[rel=canonical]');

        return null !== $canonical ? (isset($canonical->href) ? $canonical->href : '') : null;
    }

    /*
     * @return bool
     */
    public function isCanonicalCorrect()
    {
        $canonical = $this->getCanonical();

        return $canonical ? $this->response->getEffectiveUrl() == $canonical : true;
    }

    public function getKws()
    {
        $kws = TextAnalyzer::get(
            $this->response->getContent(),
            true,   // only sentences
            1,      // no expression, just words
            0      // keep trail
        );

        return $kws->getExpressions(10);
    }

    /**
     * @return int
     */
    public function getRatioTxtCode(): int
    {
        $textLenght = strlen($this->getDom()->plaintext);
        $htmlLenght = strlen(Helper::clean($this->response->getContent()));

        return (int) ($htmlLenght > 0 ? round($textLenght / $htmlLenght * 100) : 0);
    }

    /**
     * Return an array of object with two elements Link and anchor.
     *
     * @return array|null if we didn't found breadcrumb
     */
    public function getBreadCrumb(?string $separator = null)
    {
        $breadcrumb = ExtractBreadcrumb::get(
            $this->response->getContent(),
            $this->getBaseUrl(),
            $this->response->getEffectiveUrl()
        );

        if (null !== $separator && is_array($breadcrumb)) {
            $breadcrumb = array_map(function ($item) {
                return $item->getCleanName();
            }, $breadcrumb);
            $breadcrumb = implode($separator, $breadcrumb);
        }

        return $breadcrumb;
    }

    /**
     * @return string|false
     */
    public function amIRedirectToHttps()
    {
        $headers = $this->response->getHeaders();
        $headers = array_change_key_case(null !== $headers ? $headers : []);
        $redirUrl = isset($headers['location']) ? $headers['location'] : null;
        $url = $this->response->getUrl();
        if (null !== $redirUrl && ($httpsUrl = preg_replace('#^http:#', 'https:', $url, 1)) == $redirUrl) {
            return $httpsUrl;
        }

        return false;
    }

    public function getBaseUrl()
    {
        if (!isset($this->baseUrl)) {
            $base = $this->findOne('base');
            if (null !== $base && isset($base->href) && filter_var($base->href, FILTER_VALIDATE_URL)) {
                $this->baseUrl = $base->href;
            } else {
                $this->baseUrl = $this->response->getEffectiveUrl();
            }
        }

        return $this->baseUrl;
    }

    public function getDomain()
    {
        if (!isset($this->domain)) {
            $urlParsed = parse_url($this->response->getEffectiveUrl());
            preg_match("/[^\.\/]+(\.com?)?\.[^\.\/]+$/", $urlParsed['host'], $match);
            $this->domain = $match[0];
        }

        return $this->domain;
    }

    /**
     * @return int correspond to a const from Indexable
     */
    public function isIndexable(string $userAgent = 'googlebot')
    {
        return Indexable::isIndexable($this, $userAgent);
    }

    /**
     * @return RobotsTxt|string containing the current Robots.txt or NULL if an error occured
     *                          or empty string if robots is empty file
     */
    public function getRobotsTxt()
    {
        if (null === $this->robotsTxt) {
            $url = $this->getDomainAndScheme().'/robots.txt';

            $request = new CurlRequest($url);
            $request
                ->setDefaultSpeedOptions()
                ->setDownloadOnlyIf(function ($line) {
                    return 0 === stripos(trim($line), 'content-type') && false !== stripos($line, 'text/plain');
                })
                ->setUserAgent($this->getResponse()->getRequest()->getUserAgent())
            ;
            $result = $request->exec();

            $noNeedToParse = !$result instanceof \PiedWeb\Curl\Response || empty(trim($result->getContent()));

            $this->robotsTxt = $noNeedToParse ? '' : new RobotsTxt($result->getContent());
        }

        return $this->robotsTxt;
    }

    public function getDomainAndScheme()
    {
        if (null === $this->domainWithScheme) {
            $this->domainWithScheme = self::getDomainAndSchemeFrom($this->response->getEffectiveUrl());
        }

        return $this->domainWithScheme;
    }

    public static function getDomainAndSchemeFrom(string $url)
    {
        $url = parse_url($url);

        return $url['scheme'].'://'.$url['host'];
    }
}


1		<?php
2
3		namespace PiedWeb\UrlHarvester;
4
5		use PiedWeb\Curl\Response;
6		use PiedWeb\TextAnalyzer\Analyzer as TextAnalyzer;
7		use phpuri;
8		use simple_html_dom;
9		use Spatie\Robots\RobotsTxt;
10		use PiedWeb\Curl\Request as CurlRequest;
11
12		class Harvest
13		{
14		use HarvestLinksTrait;
15
16		const LINK_SELF = 1;
17		const LINK_INTERNAL = 2;
18		const LINK_SUB = 3;
19		const LINK_EXTERNAL = 4;
20
21		/**
22		* @var Response
23		*/
24		protected $response;
25
26		/**
27		* @var simple_html_dom
28		*/
29		protected $dom;
30
31		/** @var string */
32		protected $baseUrl;
33
34		/** @var string */
35		protected $domain;
36
37		/** @var RobotsTxt\|string (empty string) */
38		protected $robotsTxt;
39
40		/** @var string */
41		private $domainWithScheme;
42
43	12	public static function fromUrl(
44		string $url,
45		string $userAgent = 'Bot: Url Harvester',
46		string $language = 'en,en-US;q=0.5'
47		) {
48	12	$response = Request::make($url, $userAgent, '200;html', $language);
49
50	12	if ($response instanceof Response) {
51	12	return new self($response);
52		}
53
54		return $response;
55		}
56
57		/**
58		* @param Response $response
59		*/
60	12	public function __construct(Response $response)
61		{
62	12	$this->response = $response;
63	12	}
64
65	9	public function getResponse()
66		{
67	9	return $this->response;
68		}
69
70	3	public function getRedirection()
71		{
72	3	$headers = $this->response->getHeaders();
73	3	$headers = array_change_key_case($headers ? $headers : []);
74	3	if (isset($headers['location'])) {
75	3	return phpUri::parse($this->response->getEffectiveUrl())->join($headers['location']);
76		}
77
78		return false;
79		}
80
81	12	public function getDom()
82		{
83	12	if (null === $this->dom) {
84	6	$this->dom = new simple_html_dom();
85	6	$this->dom->load($this->response->getContent());
86		}
87
88	12	return $this->dom;
89		}
90
91	9	private function find($selector, $number = null)
92		{
93	9	return $this->getDom()->find($selector, $number);
94		}
95
96	9	private function findOne($selector)
97		{
98	9	return $this->find($selector, 0);
99		}
100
101		/**
102		* Return content inside a selector.
103		*
104		* @return string
105		*/
106	3	public function getTag($selector)
107		{
108	3	$found = $this->findOne($selector);
109
110	3	return null !== $found ? Helper::clean($found->innertext) : null;
111		}
112
113	3	public function getUniqueTag($selector = 'title')
114		{
115	3	$found = $this->find($selector);
116	3	if ($found) {
117	3	if (count($found) > 1) {
118		return count($found).' `'.$selector.'` !!';
119		} else {
120	3	return Helper::clean($found[0]->innertext);
121		}
122		}
123		}
124
125		/**
126		* Return content inside a meta.
127		*
128		* @return string from content attribute
129		*/
130	9	public function getMeta(string $name)
131		{
132	9	$meta = $this->findOne('meta[name='.$name.']');
133
134	9	return null !== $meta ? (isset($meta->content) ? Helper::clean($meta->content) : '') : '';
135		}
136
137		/**
138		* Renvoie le contenu de l'attribut href de la balise link rel=canonical.
139		*
140		* @return string le contenu de l'attribute href sinon NULL si la balise n'existe pas
141		*/
142	6	public function getCanonical()
143		{
144	6	$canonical = $this->findOne('link[rel=canonical]');
145
146	6	return null !== $canonical ? (isset($canonical->href) ? $canonical->href : '') : null;
147		}
148
149		/*
150		* @return bool
151		*/
152	6	public function isCanonicalCorrect()
153		{
154	6	$canonical = $this->getCanonical();
155
156	6	return $canonical ? $this->response->getEffectiveUrl() == $canonical : true;
157		}
158
159	3	public function getKws()
160		{
161	3	$kws = TextAnalyzer::get(
162	3	$this->response->getContent(),
163	3	true, // only sentences
164	3	1, // no expression, just words
165	3	0 // keep trail
166		);
167
168	3	return $kws->getExpressions(10);
169		}
170
171		/**
172		* @return int
173		*/
174	3	public function getRatioTxtCode(): int
175		{
176	3	$textLenght = strlen($this->getDom()->plaintext);
177	3	$htmlLenght = strlen(Helper::clean($this->response->getContent()));
178
179	3	return (int) ($htmlLenght > 0 ? round($textLenght / $htmlLenght * 100) : 0);
180		}
181
182		/**
183		* Return an array of object with two elements Link and anchor.
184		*
185		* @return array\|null if we didn't found breadcrumb
186		*/
187	3	public function getBreadCrumb(?string $separator = null)
188		{
189	3	$breadcrumb = ExtractBreadcrumb::get(
190	3	$this->response->getContent(),
191	3	$this->getBaseUrl(),
192	3	$this->response->getEffectiveUrl()
193		);
194
195	3	if (null !== $separator && is_array($breadcrumb)) {
196		$breadcrumb = array_map(function ($item) {
197		return $item->getCleanName();
198		}, $breadcrumb);
199		$breadcrumb = implode($separator, $breadcrumb);
200		}
201
202	3	return $breadcrumb;
203		}
204
205		/**
206		* @return string\|false
207		*/
208		public function amIRedirectToHttps()
209		{
210		$headers = $this->response->getHeaders();
211		$headers = array_change_key_case(null !== $headers ? $headers : []);
212		$redirUrl = isset($headers['location']) ? $headers['location'] : null;
213		$url = $this->response->getUrl();
214		if (null !== $redirUrl && ($httpsUrl = preg_replace('#^http:#', 'https:', $url, 1)) == $redirUrl) {
215		return $httpsUrl;
216		}
217
218		return false;
219		}
220
221	3	public function getBaseUrl()
222		{
223	3	if (!isset($this->baseUrl)) {
224	3	$base = $this->findOne('base');
225	3	if (null !== $base && isset($base->href) && filter_var($base->href, FILTER_VALIDATE_URL)) {
226		$this->baseUrl = $base->href;
227		} else {
228	3	$this->baseUrl = $this->response->getEffectiveUrl();
229		}
230		}
231
232	3	return $this->baseUrl;
233		}
234
235	6	public function getDomain()
236		{
237	6	if (!isset($this->domain)) {
238	6	$urlParsed = parse_url($this->response->getEffectiveUrl());
239	6	preg_match("/[^\.\/]+(\.com?)?\.[^\.\/]+$/", $urlParsed['host'], $match);
240	6	$this->domain = $match[0];
241		}
242
243	6	return $this->domain;
244		}
245
246		/**
247		* @return int correspond to a const from Indexable
248		*/
249	6	public function isIndexable(string $userAgent = 'googlebot')
250		{
251	6	return Indexable::isIndexable($this, $userAgent);
252		}
253
254		/**
255		* @return RobotsTxt\|string containing the current Robots.txt or NULL if an error occured
256		* or empty string if robots is empty file
257		*/
258	6	public function getRobotsTxt()
259		{
260	6	if (null === $this->robotsTxt) {
261	6	$url = $this->getDomainAndScheme().'/robots.txt';
262
263	6	$request = new CurlRequest($url);
264		$request
265	6	->setDefaultSpeedOptions()
266		->setDownloadOnlyIf(function ($line) {
267	6	return 0 === stripos(trim($line), 'content-type') && false !== stripos($line, 'text/plain');
268	6	})
269	6	->setUserAgent($this->getResponse()->getRequest()->getUserAgent())
270		;
271	6	$result = $request->exec();
272
273	6	$noNeedToParse = !$result instanceof \PiedWeb\Curl\Response \|\| empty(trim($result->getContent()));
274
275	6	$this->robotsTxt = $noNeedToParse ? '' : new RobotsTxt($result->getContent());
276		}
277
278	6	return $this->robotsTxt;
279		}
280
281	9	public function getDomainAndScheme()
282		{
283	9	if (null === $this->domainWithScheme) {
284	9	$this->domainWithScheme = self::getDomainAndSchemeFrom($this->response->getEffectiveUrl());
285		}
286
287	9	return $this->domainWithScheme;
288		}
289
290	9	public static function getDomainAndSchemeFrom(string $url)
291		{
292	9	$url = parse_url($url);
293
294	9	return $url['scheme'].'://'.$url['host'];
295		}
296		}
297

PiedWeb / UrlHarvester

Push — master ( db32c8...b7e19f )

Harvest::getBaseUrl() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like