PiedWeb\UrlHarvester\RobotsTxtTrait::getRobotsTxt()

Complexity

Conditions	6
Paths	5

Size

Total Lines	27
Code Lines	17

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	15
CRAP Score	6.0087

Importance

Changes	1
Bugs	0	Features	0

<?php
    public function getRobotsTxt()
    {
        if (null === $this->robotsTxt) {
            $url = $this->url()->getOrigin().'/robots.txt';

            $request = $this->getResponse()->getRequest();
            $userAgent = $request ? $request->getUserAgent() : Harvest::DEFAULT_USER_AGENT;

            $request = new CurlRequest($url);
            $request
                ->setDefaultSpeedOptions()
                ->setDownloadOnly('0-500000')
                ->setUserAgent($userAgent)
            ;
            $result = $request->exec();

            if (!$result instanceof \PiedWeb\Curl\Response
                || false === stripos($result->getContentType(), 'text/plain')
                || empty(trim($result->getContent()))
            ) {
                $this->robotsTxt = '';
            } else {
                $this->robotsTxt = new RobotsTxt($result->getContent());
            }
        }

        return $this->robotsTxt;
    }