PiedWeb\UrlHarvester\RobotsTxtTrait

Complexity

Total Complexity

9

Size/Duplication

Total Lines	54
Duplicated Lines	0 %

Test Coverage

Coverage

94.74%

Importance

Changes	1
Bugs	0	Features	0

2 Methods

Rating	Name	Duplication	Size	Complexity
A	getRobotsTxt()	0	27	6
A	setRobotsTxt()	0	5	3

<?php

namespace PiedWeb\UrlHarvester;

use PiedWeb\Curl\Request as CurlRequest;
use Spatie\Robots\RobotsTxt;

trait RobotsTxtTrait
{
    /** @var RobotsTxt|string (empty string) */
    protected $robotsTxt;

    //abstract public function getDomainAndScheme();

    abstract public function getResponse();

    abstract public function url();

    /**
     * @return RobotsTxt|string containing the current Robots.txt or NULL if an error occured
     *                          or empty string if robots is empty file
     */
    public function getRobotsTxt()
    {
        if (null === $this->robotsTxt) {
            $url = $this->url()->getOrigin().'/robots.txt';

            $request = $this->getResponse()->getRequest();
            $userAgent = $request ? $request->getUserAgent() : Harvest::DEFAULT_USER_AGENT;

            $request = new CurlRequest($url);
            $request
                ->setDefaultSpeedOptions()
                ->setDownloadOnly('0-500000')
                ->setUserAgent($userAgent)
            ;
            $result = $request->exec();

            if (! $result instanceof \PiedWeb\Curl\Response
                || false === stripos($result->getContentType(), 'text/plain')
                || empty(trim($result->getContent()))
            ) {
                $this->robotsTxt = '';
            } else {
                $this->robotsTxt = new RobotsTxt($result->getContent());
            }
        }

        return $this->robotsTxt;
    }

    /**
     * @param RobotsTxt|string $robotsTxt
     *
     * @return self
     */
    public function setRobotsTxt($robotsTxt)
    {
        $this->robotsTxt = \is_string($robotsTxt) ? (empty($robotsTxt) ? '' : new RobotsTxt($robotsTxt)) : $robotsTxt;

        return $this;
    }
}