RobotsTxt::allows() - Code Metrics - Inspection of "Fixes (see commit msg)" - spatie/robots-txt - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#13)

by Dev

created 2019-01-11 15:59 UTC

RobotsTxt::allows() B

↳ Parent: RobotsTxt

Complexity

Conditions	10
Paths	14

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	44
rs	7.6666
c	0
b	0
f	0
cc	10
nc	14
nop	2

How to fix Complexity

<?php

namespace Spatie\Robots;

class RobotsTxt
{
    protected static $robotsCache = [];

    protected $disallowsPerUserAgent = [];

    public static function readFrom(string $source): self
    {
        $content = @file_get_contents($source) ?? '';

        return new self($content);
    }

    public function __construct(string $content)
    {
        $this->disallowsPerUserAgent = $this->getDisallowsPerUserAgent($content);
    }

    public static function create(string $source): self
    {
        if (
            strpos($source, 'http') !== false
            && strpos($source, 'robots.txt') !== false
        ) {
            return self::readFrom($source);
        }

        return new self($source);
    }

    public function allows(string $url, ?string $userAgent = '*'): bool
    {
        $userAgent = strtolower($userAgent);

        if ($userAgent === null) {
            $userAgent = '*';
        }

        $path = parse_url($url, PHP_URL_PATH) ?? '';

        $disallows = $this->disallowsPerUserAgent[$userAgent] ?? $this->disallowsPerUserAgent['*'] ?? [];

        // check for exact

        $type = null;
        if ($return = $this->pathIsDenied($path, $disallows, $type) !== null) {

            // if it's in a dir, maybe wildcard authorize or forbid it
            if ($type === 1 && $wildCardReturn = $this->checkForWildcard($url, $userAgent) !== null) {
                return ! $wildCardReturn;
            }

            // if it's in a dir but wildcard forbid access
            if ($type === 1 && $wildCardReturn = $this->pathIsDenied($path, $disallows) !== null) {
                if ($wildCardReturn === true) {
                    return false;
                }
            }

            return ! $return;
        }

        // check for wildcrad user agent
        if ($return = $this->checkForWildcard($url, $userAgent) !== null) {
            return ! $return;
        }

        // check for wildcard
        if ($return = $this->pathIsDenied($path, $disallows) !== null) {
            return ! $return;
        }

        return true;
    }


    protected function checkForWildcard(string $path, string $userAgent)
    {
        if ($userAgent !== '*') {
            for ($i = 1; $i <= strlen($userAgent); $i++) {
                $wildCardUserAgent = substr($userAgent, 0, $i).'*';
                if (isset($this->disallowsPerUserAgent[$wildCardUserAgent])) {
                    return ! $this->pathIsDenied($path, $this->disallowsPerUserAgent[$wildCardUserAgent]);
                }
            }
        }
    }

    protected function pathIsDenied(string $path, array $rules, &$type = null)
    {
        foreach ($rules as $uri => $rule) {
            $trimUri = rtrim($uri, '/');

            if (in_array($path, [$uri, $trimUri])) {
                $type = 0;
                return $rule;
            }

            if (! $this->concernsDirectory($uri)) {
                $type = 0;
                return $rule;
            }

            if ($this->isUrlInDirectory($path, $uri)) {
                $type = 1;
                return $rule;
            }
        }
    }

    protected function getDisallowsPerUserAgent(string $content): array
    {
        $lines = explode(PHP_EOL, $content);

        $lines = array_filter($lines);

        $disallowsPerUserAgent = [];

        $currentUserAgent = null;

        foreach ($lines as $line) {
            if ($this->isUserAgentLine($line)) {
                $disallowsPerUserAgent[$this->parseUserAgent($line)] = [];

                $currentUserAgent = &$disallowsPerUserAgent[$this->parseUserAgent($line)];

                continue;
            }

            if ($currentUserAgent === null) {
                continue;
            }

            $rule = null;
            $disallowUrl = $this->parse($line, $rule);

            if ($disallowUrl !== null) { // other than allow/disallow
                $currentUserAgent[$disallowUrl] = $rule;
            }
        }

        return $disallowsPerUserAgent;
    }

    protected function isUserAgentLine(string $line): bool
    {
        return preg_match('/^User-agent\s*:/i', trim($line));
    }

    protected function parseUserAgent(string $line): string
    {
        return strtolower(trim(preg_replace('/^User-agent\s*:/i', '', trim($line))));
    }

    protected function parse(string $line, &$type): ?string
    {
        $line = trim(preg_replace('/\s+!/', ':', $line));

        if (stripos($line, 'disallow:') === 0) {
            $type = true;
            return trim(preg_replace('/^disallow:/i', '', $line));
        }

        if (stripos($line, 'allow:') === 0) {
            $type = false;
            return trim(preg_replace('/^allow:/i', '', $line));
        }

        // else: could be crawl-delay, sitemap...
    }

    protected function concernsDirectory(string $path): bool
    {
        return substr($path, strlen($path) - 1, 1) === '/';
    }

    protected function isUrlInDirectory(string $url, string $path): bool
    {
        return strpos($url, $path) === 0;
    }
}


1			<?php
2
3			namespace Spatie\Robots;
4
5			class RobotsTxt
6			{
7			protected static $robotsCache = [];
8
9			protected $disallowsPerUserAgent = [];
10
11			public static function readFrom(string $source): self
12			{
13			$content = @file_get_contents($source) ?? '';
14
15			return new self($content);
16			}
17
18			public function __construct(string $content)
19			{
20			$this->disallowsPerUserAgent = $this->getDisallowsPerUserAgent($content);
21			}
22
23			public static function create(string $source): self
24			{
25			if (
26			strpos($source, 'http') !== false
27			&& strpos($source, 'robots.txt') !== false
28			) {
29			return self::readFrom($source);
30			}
31
32			return new self($source);
33			}
34
35			public function allows(string $url, ?string $userAgent = '*'): bool
36			{
37			$userAgent = strtolower($userAgent);
38
39			if ($userAgent === null) {
40			$userAgent = '*';
41			}
42
43			$path = parse_url($url, PHP_URL_PATH) ?? '';
44
45			$disallows = $this->disallowsPerUserAgent[$userAgent] ?? $this->disallowsPerUserAgent['*'] ?? [];
46
47			// check for exact
48
49			$type = null;
50			if ($return = $this->pathIsDenied($path, $disallows, $type) !== null) {
51
52			// if it's in a dir, maybe wildcard authorize or forbid it
53			if ($type === 1 && $wildCardReturn = $this->checkForWildcard($url, $userAgent) !== null) {
54			return ! $wildCardReturn;
55			}
56
57			// if it's in a dir but wildcard forbid access
58			if ($type === 1 && $wildCardReturn = $this->pathIsDenied($path, $disallows) !== null) {
59			if ($wildCardReturn === true) {
60			return false;
61			}
62			}
63
64			return ! $return;
65			}
66
67			// check for wildcrad user agent
68			if ($return = $this->checkForWildcard($url, $userAgent) !== null) {
69			return ! $return;
70			}
71
72			// check for wildcard
73			if ($return = $this->pathIsDenied($path, $disallows) !== null) {
74			return ! $return;
75			}
76
77			return true;
78			}
79
80
81			protected function checkForWildcard(string $path, string $userAgent)
82			{
83			if ($userAgent !== '*') {
84			for ($i = 1; $i <= strlen($userAgent); $i++) {
85			$wildCardUserAgent = substr($userAgent, 0, $i).'*';
86			if (isset($this->disallowsPerUserAgent[$wildCardUserAgent])) {
87			return ! $this->pathIsDenied($path, $this->disallowsPerUserAgent[$wildCardUserAgent]);
88			}
89			}
90			}
91			}
92
93			protected function pathIsDenied(string $path, array $rules, &$type = null)
94			{
95			foreach ($rules as $uri => $rule) {
96			$trimUri = rtrim($uri, '/');
97
98			if (in_array($path, [$uri, $trimUri])) {
99			$type = 0;
100			return $rule;
101			}
102
103			if (! $this->concernsDirectory($uri)) {
104			$type = 0;
105			return $rule;
106			}
107
108			if ($this->isUrlInDirectory($path, $uri)) {
109			$type = 1;
110			return $rule;
111			}
112			}
113			}
114
115			protected function getDisallowsPerUserAgent(string $content): array
116			{
117			$lines = explode(PHP_EOL, $content);
118
119			$lines = array_filter($lines);
120
121			$disallowsPerUserAgent = [];
122
123			$currentUserAgent = null;
124
125			foreach ($lines as $line) {
126			if ($this->isUserAgentLine($line)) {
127			$disallowsPerUserAgent[$this->parseUserAgent($line)] = [];
128
129			$currentUserAgent = &$disallowsPerUserAgent[$this->parseUserAgent($line)];
130
131			continue;
132			}
133
134			if ($currentUserAgent === null) {
135			continue;
136			}
137
138			$rule = null;
139			$disallowUrl = $this->parse($line, $rule);
140
141			if ($disallowUrl !== null) { // other than allow/disallow
142			$currentUserAgent[$disallowUrl] = $rule;
143			}
144			}
145
146			return $disallowsPerUserAgent;
147			}
148
149			protected function isUserAgentLine(string $line): bool
150			{
151			return preg_match('/^User-agent\s*:/i', trim($line));
152			}
153
154			protected function parseUserAgent(string $line): string
155			{
156			return strtolower(trim(preg_replace('/^User-agent\s*:/i', '', trim($line))));
157			}
158
159			protected function parse(string $line, &$type): ?string
160			{
161			$line = trim(preg_replace('/\s+!/', ':', $line));
162
163			if (stripos($line, 'disallow:') === 0) {
164			$type = true;
165			return trim(preg_replace('/^disallow:/i', '', $line));
166			}
167
168			if (stripos($line, 'allow:') === 0) {
169			$type = false;
170			return trim(preg_replace('/^allow:/i', '', $line));
171			}
172
173			// else: could be crawl-delay, sitemap...
174			}
175
176			protected function concernsDirectory(string $path): bool
177			{
178			return substr($path, strlen($path) - 1, 1) === '/';
179			}
180
181			protected function isUrlInDirectory(string $url, string $path): bool
182			{
183			return strpos($url, $path) === 0;
184			}
185			}
186

GitHub Access Token became invalid

Pull Request — master (#13)

RobotsTxt::allows() B

Complexity

Size

Duplication

Importance

How to fix Complexity

Long Method

spatie / robots-txt

GitHub Access Token became invalid

Pull Request — master (#13)

RobotsTxt::allows() B

Complexity

Size

Duplication

Importance

How to fix Complexity

Long Method

Duplication Side-by-Side

Filter issues like