Spatie\Robots\RobotsTxt

Complexity

Total Complexity

36

Size/Duplication

Total Lines	184
Duplicated Lines	0 %

Coupling/Cohesion

Components	2
Dependencies	0

Importance

Changes

0

12 Methods

Rating	Name	Duplication	Size	Complexity
A	readFrom()	0	6	1
A	__construct()	0	4	1
A	create()	0	11	3
A	concernsDirectory()	0	4	1
A	isUrlInDirectory()	0	4	1
B	allows()	0	43	10
A	checkForWildcard()	0	11	4
A	pathIsDenied()	0	24	5
A	getDisallowsPerUserAgent()	0	33	5
A	isUserAgentLine()	0	4	1
A	parseUserAgent()	0	4	1
A	parse()	0	18	3

<?php

namespace Spatie\Robots;

class RobotsTxt
{
    protected static $robotsCache = [];

    protected $disallowsPerUserAgent = [];

    public static function readFrom(string $source): self
    {
        $content = @file_get_contents($source) ?? '';

        return new self($content);
    }

    public function __construct(string $content)
    {
        $this->disallowsPerUserAgent = $this->getDisallowsPerUserAgent($content);
    }

    public static function create(string $source): self
    {
        if (
            strpos($source, 'http') !== false
            && strpos($source, 'robots.txt') !== false
        ) {
            return self::readFrom($source);
        }

        return new self($source);
    }

    public function allows(string $url, ?string $userAgent = '*'): bool
    {
        $userAgent = strtolower($userAgent);

        if ($userAgent === null) {
            $userAgent = '*';
        }

        $path = parse_url($url, PHP_URL_PATH) ?? '';

        $disallows = $this->disallowsPerUserAgent[$userAgent] ?? $this->disallowsPerUserAgent['*'] ?? [];

        // check for exact
        $type = null;
        if ($return = $this->pathIsDenied($path, $disallows, $type) !== null) {

            // if it's in a dir, maybe wildcard authorize or forbid it
            if ($type === 1 && $wildCardReturn = $this->checkForWildcard($url, $userAgent) !== null) {
                return $wildCardReturn;
            }

            // if it's in a dir but wildcard forbid access
            if ($type === 1 && $wildCardReturn = $this->pathIsDenied($path, $disallows) !== null) {
                if ($wildCardReturn === true) {
                    return false;
                }
            }

            return ! $return;
        }

        // check for wildcrad user agent
        if ($return = $this->checkForWildcard($url, $userAgent) !== null) {
            return ! $return;
        }

        // check for wildcard
        if ($return = $this->pathIsDenied($path, $disallows) !== null) {
            return ! $return;
        }

        return true;
    }

    protected function checkForWildcard(string $path, string $userAgent)
    {
        if ($userAgent !== '*') {
            for ($i = 1; $i <= strlen($userAgent); $i++) {
                $wildCardUserAgent = substr($userAgent, 0, $i).'*';
                if (isset($this->disallowsPerUserAgent[$wildCardUserAgent])) {
                    return ! $this->pathIsDenied($path, $this->disallowsPerUserAgent[$wildCardUserAgent]);
                }
            }
        }
    }

    protected function pathIsDenied(string $path, array $rules, &$type = null)
    {
        foreach ($rules as $uri => $rule) {
            $trimUri = rtrim($uri, '/');

            if (in_array($path, [$uri, $trimUri])) {
                $type = 0;

                return $rule;
            }

            if (! $this->concernsDirectory($uri)) {
                $type = 0;

                return $rule;
            }

            if ($this->isUrlInDirectory($path, $uri)) {
                $type = 1;

                return $rule;
            }
        }
    }

    protected function getDisallowsPerUserAgent(string $content): array
    {
        $lines = explode(PHP_EOL, $content);

        $lines = array_filter($lines);

        $disallowsPerUserAgent = [];

        $currentUserAgent = null;

        foreach ($lines as $line) {
            if ($this->isUserAgentLine($line)) {
                $disallowsPerUserAgent[$this->parseUserAgent($line)] = [];

                $currentUserAgent = &$disallowsPerUserAgent[$this->parseUserAgent($line)];

                continue;
            }

            if ($currentUserAgent === null) {
                continue;
            }

            $rule = null;
            $disallowUrl = $this->parse($line, $rule);

            if ($disallowUrl !== null) { // other than allow/disallow
                $currentUserAgent[$disallowUrl] = $rule;
            }
        }

        return $disallowsPerUserAgent;
    }

    protected function isUserAgentLine(string $line): bool
    {
        return preg_match('/^User-agent\s*:/i', trim($line));
    }

    protected function parseUserAgent(string $line): string
    {
        return strtolower(trim(preg_replace('/^User-agent\s*:/i', '', trim($line))));
    }

    protected function parse(string $line, &$type): ?string
    {
        $line = trim(preg_replace('/\s+!/', ':', $line));

        if (stripos($line, 'disallow:') === 0) {
            $type = true;

            return trim(preg_replace('/^disallow:/i', '', $line));
        }

        if (stripos($line, 'allow:') === 0) {
            $type = false;

            return trim(preg_replace('/^allow:/i', '', $line));
        }

        // else: could be crawl-delay, sitemap...
    }

    protected function concernsDirectory(string $path): bool
    {
        return substr($path, strlen($path) - 1, 1) === '/';
    }

    protected function isUrlInDirectory(string $url, string $path): bool
    {
        return strpos($url, $path) === 0;
    }
}