Spatie\Crawler\Handlers\CrawlRequestFulfilled

Complexity

Total Complexity

17

Size/Duplication

Total Lines	127
Duplicated Lines	25.2 %

Coupling/Cohesion

Components	2
Dependencies	8

Importance

Changes

0

6 Methods

Rating	Name	Duplication	Size	Complexity
A	__construct()	0	17	1
B	__invoke()	0	31	5
A	convertBodyToString()	0	8	1
A	handleCrawled()	0	10	2
A	mayIndex()	16	16	4
A	mayFollow()	16	16	4

<?php

namespace Spatie\Crawler\Handlers;

use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\StreamInterface;
use Psr\Http\Message\UriInterface;
use Spatie\Crawler\Crawler;
use Spatie\Crawler\CrawlProfile;
use Spatie\Crawler\CrawlQueue\CrawlQueue;
use Spatie\Crawler\CrawlSubdomains;
use Spatie\Crawler\CrawlUrl;
use Spatie\Robots\RobotsHeaders;
use Spatie\Robots\RobotsMeta;

class CrawlRequestFulfilled
{
    /** @var \Spatie\Crawler\Crawler */
    protected $crawler;

    /** @var \Spatie\Crawler\CrawlQueue\CrawlQueue */
    protected $crawlQueue;

    /** @var \Spatie\Crawler\CrawlProfile */
    protected $crawlProfile;

    /** @var array[\Spatie\Crawler\CrawlObserver] */
    protected $crawlObservers;

    /** @var \Psr\Http\Message\UriInterface */
    protected $baseUrl;

    /** @var int */
    protected $maximumResponseSize;

    /** @var bool */
    protected $respectRobots;

    public function __construct(
        Crawler $crawler,
        UriInterface $baseUrl,
        CrawlQueue $crawlQueue,
        CrawlProfile $crawlProfile,
        array $crawlObservers,
        int $maximumResponseSize,
        bool $respectRobots
    ) {
        $this->crawler = $crawler;
        $this->baseUrl = $baseUrl;
        $this->crawlQueue = $crawlQueue;
        $this->crawlProfile = $crawlProfile;
        $this->crawlObservers = $crawlObservers;
        $this->maximumResponseSize = $maximumResponseSize;
        $this->respectRobots = $respectRobots;
    }

    public function __invoke(ResponseInterface $response, $index)
    {
        $crawlUrl = $this->crawlQueue->getUrlById($index);

        $body = $this->convertBodyToString($response->getBody(), $this->maximumResponseSize);

        $robotsHeaders = RobotsHeaders::create($response->getHeaders());

        $robotsMeta = RobotsMeta::create($body);

        if (! $this->mayIndex($robotsHeaders, $robotsMeta)) {
            return;
        }

        $this->handleCrawled($response, $crawlUrl);

        if (! $this->crawlProfile instanceof CrawlSubdomains) {
            if ($crawlUrl->url->getHost() !== $this->baseUrl->getHost()) {
                return;
            }
        }

        if (! $this->mayFollow($robotsHeaders, $robotsMeta)) {
            return;
        }

        $this->crawler->addAllLinksToCrawlQueue(
            $body,
            $crawlUrl->url
        );
    }

    protected function convertBodyToString(StreamInterface $bodyStream, $readMaximumBytes = 1024 * 1024 * 2): string
    {
        $bodyStream->rewind();

        $body = $bodyStream->read($readMaximumBytes);

        return $body;
    }

    protected function handleCrawled(ResponseInterface $response, CrawlUrl $crawlUrl)
    {
        foreach ($this->crawlObservers as $crawlObserver) {
            $crawlObserver->crawled(
                $crawlUrl->url,
                $response,
                $crawlUrl->foundOnUrl
            );
        }
    }

    protected function mayIndex(RobotsHeaders $robotsHeaders, RobotsMeta $robotsMeta): bool

This method seems to be duplicated in your project.

    {
        if (! $this->respectRobots) {
            return true;
        }

        if (! $robotsHeaders->mayIndex()) {
            return false;
        }

        if (! $robotsMeta->mayIndex()) {
            return false;
        }

        return true;
    }

    protected function mayFollow(RobotsHeaders $robotsHeaders, RobotsMeta $robotsMeta): bool

This method seems to be duplicated in your project.

    {
        if (! $this->respectRobots) {
            return true;
        }

        if (! $robotsHeaders->mayFollow()) {
            return false;
        }

        if (! $robotsMeta->mayFollow()) {
            return false;
        }

        return true;
    }
}