CrawlLogger::finishedCrawling() - Code Metrics - Inspection of "WIP: Output full crawl log to CSV file" - spatie/http-status-check - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#67)

unknown

created 2020-02-17 20:01 UTC

CrawlLogger::finishedCrawling() A

↳ Parent: CrawlLogger

Complexity

Conditions	5
Paths	10

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	28
rs	9.1608
c	0
b	0
f	0
cc	5
nc	10
nop	0

<?php

namespace Spatie\HttpStatusCheck;

use GuzzleHttp\Exception\RequestException;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\UriInterface;
use Spatie\Crawler\CrawlObserver;
use Symfony\Component\Console\Output\OutputInterface;

class CrawlLogger extends CrawlObserver
{
    const UNRESPONSIVE_HOST = 'Host did not respond';

    /**
     * @var \Symfony\Component\Console\Output\OutputInterface
     */
    protected $consoleOutput;

    /**
     * @var array
     */
    protected $crawledUrls = [];

    /**
     * @var string|null
     */
    protected $outputFile = null;

    /**
     * @var resource|null
     */
    protected $csvFile = null;

    /**
     * @param \Symfony\Component\Console\Output\OutputInterface $consoleOutput
     */
    public function __construct(OutputInterface $consoleOutput)
    {
        $this->consoleOutput = $consoleOutput;
    }

    /**
     * Called when the crawl will crawl the url.
     *
     * @param \Psr\Http\Message\UriInterface $url
     */
    public function willCrawl(UriInterface $url)
    {
    }

    /**
     * Called when the crawl has ended.
     */
    public function finishedCrawling()
    {
        if ($this->csvFile) {
            fclose($this->csvFile);
        }

        $this->consoleOutput->writeln('');
        $this->consoleOutput->writeln('Crawling summary');
        $this->consoleOutput->writeln('----------------');

        ksort($this->crawledUrls);

        foreach ($this->crawledUrls as $statusCode => $urls) {
            $colorTag = $this->getColorTagForStatusCode($statusCode);

            $count = count($urls);

            if (is_numeric($statusCode)) {
                $this->consoleOutput->writeln("<{$colorTag}>Crawled {$count} url(s) with statuscode {$statusCode}</{$colorTag}>");
            }

            if ($statusCode == static::UNRESPONSIVE_HOST) {
                $this->consoleOutput->writeln("<{$colorTag}>{$count} url(s) did have unresponsive host(s)</{$colorTag}>");
            }
        }

        $this->consoleOutput->writeln('');
    }

    protected function getColorTagForStatusCode(string $code): string
    {
        if ($this->startsWith($code, '2')) {
            return 'info';
        }

        if ($this->startsWith($code, '3')) {
            return 'comment';
        }

        return 'error';
    }

    /**
     * @param string|null $haystack
     * @param string|array $needles
     *
     * @return bool
     */
    public function startsWith($haystack, $needles): bool
    {
        foreach ((array) $needles as $needle) {
            if ($needle != '' && substr($haystack, 0, strlen($needle)) === (string) $needle) {
                return true;
            }
        }

        return false;
    }

    /**
     * Set the filename to write the output log.
     *
     * @param string $filename
     */
    public function setOutputFile($filename)
    {
        $this->outputFile = $filename;
    }

    /**
     * Set the filename (and open the file) to write the csv log.
     *
     * @param string $filename
     */
    public function setCsvFile($filename)
    {
        $this->csvFile = fopen($filename, 'w');
    }

    public function crawled(
        UriInterface $url,
        ResponseInterface $response,
        ?UriInterface $foundOnUrl = null
    ) {
        $statusCode = $response->getStatusCode();

        $reason = $response->getReasonPhrase();

        $colorTag = $this->getColorTagForStatusCode($statusCode);

        $timestamp = date('Y-m-d H:i:s');

        $message = "{$statusCode} {$reason} - ".(string) $url;

        if ($this->outputFile && $colorTag === 'error') {
''   == false // true
''   == null  // true
'ab' == false // false
'ab' == null  // false

// It is often better to use strict comparison
'' === false // false
'' === null  // false
            file_put_contents($this->outputFile, $message.PHP_EOL, FILE_APPEND);
        }

        if ($this->csvFile) {
            fputcsv($this->csvFile, [$timestamp, $statusCode, $reason, (string) $url, (string) $foundOnUrl]);
        }

        $this->consoleOutput->writeln("<{$colorTag}>[{$timestamp}] {$message}</{$colorTag}>");

        $this->crawledUrls[$statusCode][] = $url;
    }

    public function crawlFailed(
        UriInterface $url,
        RequestException $requestException,
        ?UriInterface $foundOnUrl = null
    ) {
        $statusCode = $requestException->getResponse()
            ? $requestException->getResponse()->getStatusCode()
            : self::UNRESPONSIVE_HOST;

        $reason = $requestException->getResponse()
            ? $requestException->getResponse()->getReasonPhrase()
            : $requestException->getMessage();

        $colorTag = $this->getColorTagForStatusCode($statusCode);

        $timestamp = date('Y-m-d H:i:s');

        $message = "{$statusCode}: {$reason} - ".(string) $url;

        if ($foundOnUrl) {
            $message .= " (found on {$foundOnUrl})";
        }

        if ($this->outputFile) {
''   == false // true
''   == null  // true
'ab' == false // false
'ab' == null  // false

// It is often better to use strict comparison
'' === false // false
'' === null  // false
            file_put_contents($this->outputFile, $message.PHP_EOL, FILE_APPEND);
        }

        if ($this->csvFile) {
            fputcsv($this->csvFile, [$timestamp, $statusCode, $reason, (string) $url, (string) $foundOnUrl]);
        }

        $this->consoleOutput->writeln("<{$colorTag}>[{$timestamp}] {$message}</{$colorTag}>");

        $this->crawledUrls[$statusCode][] = $url;
    }
}


1			<?php
2
3			namespace Spatie\HttpStatusCheck;
4
5			use GuzzleHttp\Exception\RequestException;
6			use Psr\Http\Message\ResponseInterface;
7			use Psr\Http\Message\UriInterface;
8			use Spatie\Crawler\CrawlObserver;
9			use Symfony\Component\Console\Output\OutputInterface;
10
11			class CrawlLogger extends CrawlObserver
12			{
13			const UNRESPONSIVE_HOST = 'Host did not respond';
14
15			/**
16			* @var \Symfony\Component\Console\Output\OutputInterface
17			*/
18			protected $consoleOutput;
19
20			/**
21			* @var array
22			*/
23			protected $crawledUrls = [];
24
25			/**
26			* @var string\|null
27			*/
28			protected $outputFile = null;
29
30			/**
31			* @var resource\|null
32			*/
33			protected $csvFile = null;
34
35			/**
36			* @param \Symfony\Component\Console\Output\OutputInterface $consoleOutput
37			*/
38			public function __construct(OutputInterface $consoleOutput)
39			{
40			$this->consoleOutput = $consoleOutput;
41			}
42
43			/**
44			* Called when the crawl will crawl the url.
45			*
46			* @param \Psr\Http\Message\UriInterface $url
47			*/
48			public function willCrawl(UriInterface $url)
49			{
50			}
51
52			/**
53			* Called when the crawl has ended.
54			*/
55			public function finishedCrawling()
56			{
57			if ($this->csvFile) {
58			fclose($this->csvFile);
59			}
60
61			$this->consoleOutput->writeln('');
62			$this->consoleOutput->writeln('Crawling summary');
63			$this->consoleOutput->writeln('----------------');
64
65			ksort($this->crawledUrls);
66
67			foreach ($this->crawledUrls as $statusCode => $urls) {
68			$colorTag = $this->getColorTagForStatusCode($statusCode);
69
70			$count = count($urls);
71
72			if (is_numeric($statusCode)) {
73			$this->consoleOutput->writeln("<{$colorTag}>Crawled {$count} url(s) with statuscode {$statusCode}</{$colorTag}>");
74			}
75
76			if ($statusCode == static::UNRESPONSIVE_HOST) {
77			$this->consoleOutput->writeln("<{$colorTag}>{$count} url(s) did have unresponsive host(s)</{$colorTag}>");
78			}
79			}
80
81			$this->consoleOutput->writeln('');
82			}
83
84			protected function getColorTagForStatusCode(string $code): string
85			{
86			if ($this->startsWith($code, '2')) {
87			return 'info';
88			}
89
90			if ($this->startsWith($code, '3')) {
91			return 'comment';
92			}
93
94			return 'error';
95			}
96
97			/**
98			* @param string\|null $haystack
99			* @param string\|array $needles
100			*
101			* @return bool
102			*/
103			public function startsWith($haystack, $needles): bool
104			{
105			foreach ((array) $needles as $needle) {
106			if ($needle != '' && substr($haystack, 0, strlen($needle)) === (string) $needle) {
107			return true;
108			}
109			}
110
111			return false;
112			}
113
114			/**
115			* Set the filename to write the output log.
116			*
117			* @param string $filename
118			*/
119			public function setOutputFile($filename)
120			{
121			$this->outputFile = $filename;
122			}
123
124			/**
125			* Set the filename (and open the file) to write the csv log.
126			*
127			* @param string $filename
128			*/
129			public function setCsvFile($filename)
130			{
131			$this->csvFile = fopen($filename, 'w');
132			}
133
134			public function crawled(
135			UriInterface $url,
136			ResponseInterface $response,
137			?UriInterface $foundOnUrl = null
138			) {
139			$statusCode = $response->getStatusCode();
140
141			$reason = $response->getReasonPhrase();
142
143			$colorTag = $this->getColorTagForStatusCode($statusCode);
144
145			$timestamp = date('Y-m-d H:i:s');
146
147			$message = "{$statusCode} {$reason} - ".(string) $url;
148
149			if ($this->outputFile && $colorTag === 'error') {
			0 ignored issues – show Bug Best Practice introduced 2016-12-16 16:16 UTC by Report Bug Copy Issue Report The expression `$this->outputFile` of type `string\|null` is loosely compared to `true`; this is ambiguous if the string can be empty. You might want to explicitly use `!== null` instead. In PHP, under loose comparison (like `==`, or `!=`, or `switch` conditions), values of different types might be equal. For `string` values, the empty string `''` is a special case, in particular the following results might be unexpected: '' == false // true '' == null // true 'ab' == false // false 'ab' == null // false // It is often better to use strict comparison '' === false // false '' === null // false Loading history...
150			file_put_contents($this->outputFile, $message.PHP_EOL, FILE_APPEND);
151			}
152
153			if ($this->csvFile) {
154			fputcsv($this->csvFile, [$timestamp, $statusCode, $reason, (string) $url, (string) $foundOnUrl]);
155			}
156
157			$this->consoleOutput->writeln("<{$colorTag}>[{$timestamp}] {$message}</{$colorTag}>");
158
159			$this->crawledUrls[$statusCode][] = $url;
160			}
161
162			public function crawlFailed(
163			UriInterface $url,
164			RequestException $requestException,
165			?UriInterface $foundOnUrl = null
166			) {
167			$statusCode = $requestException->getResponse()
168			? $requestException->getResponse()->getStatusCode()
169			: self::UNRESPONSIVE_HOST;
170
171			$reason = $requestException->getResponse()
172			? $requestException->getResponse()->getReasonPhrase()
173			: $requestException->getMessage();
174
175			$colorTag = $this->getColorTagForStatusCode($statusCode);
176
177			$timestamp = date('Y-m-d H:i:s');
178
179			$message = "{$statusCode}: {$reason} - ".(string) $url;
180
181			if ($foundOnUrl) {
182			$message .= " (found on {$foundOnUrl})";
183			}
184
185			if ($this->outputFile) {
			0 ignored issues – show Bug Best Practice introduced 2018-05-09 14:39 UTC by Report Bug Copy Issue Report The expression `$this->outputFile` of type `string\|null` is loosely compared to `true`; this is ambiguous if the string can be empty. You might want to explicitly use `!== null` instead. In PHP, under loose comparison (like `==`, or `!=`, or `switch` conditions), values of different types might be equal. For `string` values, the empty string `''` is a special case, in particular the following results might be unexpected: '' == false // true '' == null // true 'ab' == false // false 'ab' == null // false // It is often better to use strict comparison '' === false // false '' === null // false Loading history...
186			file_put_contents($this->outputFile, $message.PHP_EOL, FILE_APPEND);
187			}
188
189			if ($this->csvFile) {
190			fputcsv($this->csvFile, [$timestamp, $statusCode, $reason, (string) $url, (string) $foundOnUrl]);
191			}
192
193			$this->consoleOutput->writeln("<{$colorTag}>[{$timestamp}] {$message}</{$colorTag}>");
194
195			$this->crawledUrls[$statusCode][] = $url;
196			}
197			}
198

spatie / http-status-check

GitHub Access Token became invalid

Pull Request — master (#67)

CrawlLogger::finishedCrawling() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like