CrawlLogger::finishedCrawling() - Code Metrics - Inspection of "follow and log redirects as multiple responses (#6..." - spatie/http-status-check - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( ecd136...6b9366 )

by Freek

created 2020-03-18 21:11 UTC

CrawlLogger::finishedCrawling() A

↳ Parent: CrawlLogger

Complexity

Conditions	4
Paths	5

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	4
nc	5
nop	0
dl	0
loc	24
rs	9.536
c	0
b	0
f	0

<?php

namespace Spatie\HttpStatusCheck;

use GuzzleHttp\Exception\RequestException;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\UriInterface;
use Spatie\Crawler\CrawlObserver;
use Symfony\Component\Console\Output\OutputInterface;

class CrawlLogger extends CrawlObserver
{
    const UNRESPONSIVE_HOST = 'Host did not respond';
    const REDIRECT = 'Redirect';

    /**
     * @var \Symfony\Component\Console\Output\OutputInterface
     */
    protected $consoleOutput;

    /**
     * @var array
     */
    protected $crawledUrls = [];

    /**
     * @var string|null
     */
    protected $outputFile = null;

    /**
     * @param \Symfony\Component\Console\Output\OutputInterface $consoleOutput
     */
    public function __construct(OutputInterface $consoleOutput)
    {
        $this->consoleOutput = $consoleOutput;
    }

    /**
     * Called when the crawl will crawl the url.
     *
     * @param \Psr\Http\Message\UriInterface $url
     */
    public function willCrawl(UriInterface $url)
    {
    }

    /**
     * Called when the crawl has ended.
     */
    public function finishedCrawling()
    {
        $this->consoleOutput->writeln('');
        $this->consoleOutput->writeln('Crawling summary');
        $this->consoleOutput->writeln('----------------');

        ksort($this->crawledUrls);

        foreach ($this->crawledUrls as $statusCode => $urls) {
            $colorTag = $this->getColorTagForStatusCode($statusCode);

            $count = count($urls);

            if (is_numeric($statusCode)) {
                $this->consoleOutput->writeln("<{$colorTag}>Crawled {$count} url(s) with statuscode {$statusCode}</{$colorTag}>");
            }

            if ($statusCode == static::UNRESPONSIVE_HOST) {
                $this->consoleOutput->writeln("<{$colorTag}>{$count} url(s) did have unresponsive host(s)</{$colorTag}>");
            }
        }

        $this->consoleOutput->writeln('');
    }

    protected function getColorTagForStatusCode(string $code): string
    {
        if ($this->startsWith($code, '2')) {
            return 'info';
        }

        if ($this->startsWith($code, '3')) {
            return 'comment';
        }

        return 'error';
    }

    /**
     * @param string|null $haystack
     * @param string|array $needles
     *
     * @return bool
     */
    public function startsWith($haystack, $needles): bool
    {
        foreach ((array) $needles as $needle) {
            if ($needle != '' && substr($haystack, 0, strlen($needle)) === (string) $needle) {
                return true;
            }
        }

        return false;
    }

    /**
     * Set the filename to write the output log.
     *
     * @param string $filename
     */
    public function setOutputFile($filename)
    {
        $this->outputFile = $filename;
    }

    public function crawled(
        UriInterface $url,
        ResponseInterface $response,
        ?UriInterface $foundOnUrl = null
    ) {
        if ($this->addRedirectedResult($url, $response, $foundOnUrl)) {
            return;
        }

        // response wasnt a redirect so lets add it as a standard result
        $this->addResult(
            (string) $url,
            (string) $foundOnUrl,
            $response->getStatusCode(),
            $response->getReasonPhrase()
        );
    }

    public function crawlFailed(
        UriInterface $url,
        RequestException $requestException,
        ?UriInterface $foundOnUrl = null
    ) {
        if ($response = $requestException->getResponse()) {
            $this->crawled($url, $response, $foundOnUrl);
        } else {
            $this->addResult((string) $url, (string) $foundOnUrl, '---', self::UNRESPONSIVE_HOST);
        }
    }

    public function addResult($url, $foundOnUrl, $statusCode, $reason)
    {
        /*
        * don't display duplicate results
        * this happens if a redirect is followed to an existing page
        */
        if (isset($this->crawledUrls[$statusCode]) && in_array($url, $this->crawledUrls[$statusCode])) {
            return;
        }

        $colorTag = $this->getColorTagForStatusCode($statusCode);

        $timestamp = date('Y-m-d H:i:s');

        $message = "{$statusCode} {$reason} - ".(string) $url;

        if ($foundOnUrl && $colorTag === 'error') {
            $message .= " (found on {$foundOnUrl})";
        }

        if ($this->outputFile && $colorTag === 'error') {
''   == false // true
''   == null  // true
'ab' == false // false
'ab' == null  // false

// It is often better to use strict comparison
'' === false // false
'' === null  // false
            file_put_contents($this->outputFile, $message.PHP_EOL, FILE_APPEND);
        }

        $this->consoleOutput->writeln("<{$colorTag}>[{$timestamp}] {$message}</{$colorTag}>");

        $this->crawledUrls[$statusCode][] = $url;
    }

    /*
    * https://github.com/guzzle/guzzle/blob/master/docs/faq.rst#how-can-i-track-redirected-requests
    */
    public function addRedirectedResult(
        UriInterface $url,
        ResponseInterface $response,
        ?UriInterface $foundOnUrl = null
    ) {
        // if its not a redirect the return false
        if (! $response->getHeader('X-Guzzle-Redirect-History')) {
            return false;
        }

        // retrieve Redirect URI history
        $redirectUriHistory = $response->getHeader('X-Guzzle-Redirect-History');

        // retrieve Redirect HTTP Status history
        $redirectCodeHistory = $response->getHeader('X-Guzzle-Redirect-Status-History');

        // Add the initial URI requested to the (beginning of) URI history
        array_unshift($redirectUriHistory, (string) $url);

        // Add the final HTTP status code to the end of HTTP response history
        array_push($redirectCodeHistory, $response->getStatusCode());

        // Combine the items of each array into a single result set
        $fullRedirectReport = [];
        foreach ($redirectUriHistory as $key => $value) {
            $fullRedirectReport[$key] = ['location' => $value, 'code' => $redirectCodeHistory[$key]];
        }

        // Add the redirects and final URL as results
        foreach ($fullRedirectReport as $k=>$redirect) {
            $this->addResult(
                (string) $redirect['location'],
                (string) $foundOnUrl,
                $redirect['code'],
                $k + 1 == count($fullRedirectReport) ? $response->getReasonPhrase() : self::REDIRECT
            );
        }

        return true;
    }
}


1			<?php
2
3			namespace Spatie\HttpStatusCheck;
4
5			use GuzzleHttp\Exception\RequestException;
6			use Psr\Http\Message\ResponseInterface;
7			use Psr\Http\Message\UriInterface;
8			use Spatie\Crawler\CrawlObserver;
9			use Symfony\Component\Console\Output\OutputInterface;
10
11			class CrawlLogger extends CrawlObserver
12			{
13			const UNRESPONSIVE_HOST = 'Host did not respond';
14			const REDIRECT = 'Redirect';
15
16			/**
17			* @var \Symfony\Component\Console\Output\OutputInterface
18			*/
19			protected $consoleOutput;
20
21			/**
22			* @var array
23			*/
24			protected $crawledUrls = [];
25
26			/**
27			* @var string\|null
28			*/
29			protected $outputFile = null;
30
31			/**
32			* @param \Symfony\Component\Console\Output\OutputInterface $consoleOutput
33			*/
34			public function __construct(OutputInterface $consoleOutput)
35			{
36			$this->consoleOutput = $consoleOutput;
37			}
38
39			/**
40			* Called when the crawl will crawl the url.
41			*
42			* @param \Psr\Http\Message\UriInterface $url
43			*/
44			public function willCrawl(UriInterface $url)
45			{
46			}
47
48			/**
49			* Called when the crawl has ended.
50			*/
51			public function finishedCrawling()
52			{
53			$this->consoleOutput->writeln('');
54			$this->consoleOutput->writeln('Crawling summary');
55			$this->consoleOutput->writeln('----------------');
56
57			ksort($this->crawledUrls);
58
59			foreach ($this->crawledUrls as $statusCode => $urls) {
60			$colorTag = $this->getColorTagForStatusCode($statusCode);
61
62			$count = count($urls);
63
64			if (is_numeric($statusCode)) {
65			$this->consoleOutput->writeln("<{$colorTag}>Crawled {$count} url(s) with statuscode {$statusCode}</{$colorTag}>");
66			}
67
68			if ($statusCode == static::UNRESPONSIVE_HOST) {
69			$this->consoleOutput->writeln("<{$colorTag}>{$count} url(s) did have unresponsive host(s)</{$colorTag}>");
70			}
71			}
72
73			$this->consoleOutput->writeln('');
74			}
75
76			protected function getColorTagForStatusCode(string $code): string
77			{
78			if ($this->startsWith($code, '2')) {
79			return 'info';
80			}
81
82			if ($this->startsWith($code, '3')) {
83			return 'comment';
84			}
85
86			return 'error';
87			}
88
89			/**
90			* @param string\|null $haystack
91			* @param string\|array $needles
92			*
93			* @return bool
94			*/
95			public function startsWith($haystack, $needles): bool
96			{
97			foreach ((array) $needles as $needle) {
98			if ($needle != '' && substr($haystack, 0, strlen($needle)) === (string) $needle) {
99			return true;
100			}
101			}
102
103			return false;
104			}
105
106			/**
107			* Set the filename to write the output log.
108			*
109			* @param string $filename
110			*/
111			public function setOutputFile($filename)
112			{
113			$this->outputFile = $filename;
114			}
115
116			public function crawled(
117			UriInterface $url,
118			ResponseInterface $response,
119			?UriInterface $foundOnUrl = null
120			) {
121			if ($this->addRedirectedResult($url, $response, $foundOnUrl)) {
122			return;
123			}
124
125			// response wasnt a redirect so lets add it as a standard result
126			$this->addResult(
127			(string) $url,
128			(string) $foundOnUrl,
129			$response->getStatusCode(),
130			$response->getReasonPhrase()
131			);
132			}
133
134			public function crawlFailed(
135			UriInterface $url,
136			RequestException $requestException,
137			?UriInterface $foundOnUrl = null
138			) {
139			if ($response = $requestException->getResponse()) {
140			$this->crawled($url, $response, $foundOnUrl);
141			} else {
142			$this->addResult((string) $url, (string) $foundOnUrl, '---', self::UNRESPONSIVE_HOST);
143			}
144			}
145
146			public function addResult($url, $foundOnUrl, $statusCode, $reason)
147			{
148			/*
149			* don't display duplicate results
150			* this happens if a redirect is followed to an existing page
151			*/
152			if (isset($this->crawledUrls[$statusCode]) && in_array($url, $this->crawledUrls[$statusCode])) {
153			return;
154			}
155
156			$colorTag = $this->getColorTagForStatusCode($statusCode);
157
158			$timestamp = date('Y-m-d H:i:s');
159
160			$message = "{$statusCode} {$reason} - ".(string) $url;
161
162			if ($foundOnUrl && $colorTag === 'error') {
163			$message .= " (found on {$foundOnUrl})";
164			}
165
166			if ($this->outputFile && $colorTag === 'error') {
			0 ignored issues – show Bug Best Practice introduced 2016-12-16 16:16 UTC by Report Bug Copy Issue Report The expression `$this->outputFile` of type `string\|null` is loosely compared to `true`; this is ambiguous if the string can be empty. You might want to explicitly use `!== null` instead. In PHP, under loose comparison (like `==`, or `!=`, or `switch` conditions), values of different types might be equal. For `string` values, the empty string `''` is a special case, in particular the following results might be unexpected: '' == false // true '' == null // true 'ab' == false // false 'ab' == null // false // It is often better to use strict comparison '' === false // false '' === null // false Loading history...
167			file_put_contents($this->outputFile, $message.PHP_EOL, FILE_APPEND);
168			}
169
170			$this->consoleOutput->writeln("<{$colorTag}>[{$timestamp}] {$message}</{$colorTag}>");
171
172			$this->crawledUrls[$statusCode][] = $url;
173			}
174
175			/*
176			* https://github.com/guzzle/guzzle/blob/master/docs/faq.rst#how-can-i-track-redirected-requests
177			*/
178			public function addRedirectedResult(
179			UriInterface $url,
180			ResponseInterface $response,
181			?UriInterface $foundOnUrl = null
182			) {
183			// if its not a redirect the return false
184			if (! $response->getHeader('X-Guzzle-Redirect-History')) {
185			return false;
186			}
187
188			// retrieve Redirect URI history
189			$redirectUriHistory = $response->getHeader('X-Guzzle-Redirect-History');
190
191			// retrieve Redirect HTTP Status history
192			$redirectCodeHistory = $response->getHeader('X-Guzzle-Redirect-Status-History');
193
194			// Add the initial URI requested to the (beginning of) URI history
195			array_unshift($redirectUriHistory, (string) $url);
196
197			// Add the final HTTP status code to the end of HTTP response history
198			array_push($redirectCodeHistory, $response->getStatusCode());
199
200			// Combine the items of each array into a single result set
201			$fullRedirectReport = [];
202			foreach ($redirectUriHistory as $key => $value) {
203			$fullRedirectReport[$key] = ['location' => $value, 'code' => $redirectCodeHistory[$key]];
204			}
205
206			// Add the redirects and final URL as results
207			foreach ($fullRedirectReport as $k=>$redirect) {
208			$this->addResult(
209			(string) $redirect['location'],
210			(string) $foundOnUrl,
211			$redirect['code'],
212			$k + 1 == count($fullRedirectReport) ? $response->getReasonPhrase() : self::REDIRECT
213			);
214			}
215
216			return true;
217			}
218			}
219

spatie / http-status-check

GitHub Access Token became invalid

Push — master ( ecd136...6b9366 )

CrawlLogger::finishedCrawling() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like