|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
namespace Bee4\RobotsTxt; |
|
4
|
|
|
|
|
5
|
|
|
use Bee4\RobotsTxt\Exception\InvalidContentException; |
|
6
|
|
|
|
|
7
|
|
|
/** |
|
8
|
|
|
* Class Parser |
|
9
|
|
|
* Take the content of a robots.txt file and transform it to rules |
|
10
|
|
|
* |
|
11
|
|
|
* @copyright Bee4 2015 |
|
12
|
|
|
* @author Stephane HULARD <[email protected]> |
|
13
|
|
|
*/ |
|
14
|
|
|
class Parser |
|
15
|
|
|
{ |
|
16
|
|
|
/** |
|
17
|
|
|
* Parse the current content |
|
18
|
|
|
* @return Rules |
|
19
|
|
|
*/ |
|
20
|
|
|
public function analyze($content) |
|
21
|
|
|
{ |
|
22
|
1 |
|
$content = $this->handleContent($content); |
|
23
|
|
|
|
|
24
|
1 |
|
$rules = new Rules(); |
|
25
|
1 |
|
$current = []; |
|
26
|
1 |
|
$ua = false; |
|
27
|
|
|
|
|
28
|
1 |
|
foreach ($content as $line) { |
|
29
|
1 |
|
if (0 === strpos($line, '#')) { |
|
30
|
1 |
|
continue; |
|
31
|
|
|
} |
|
32
|
|
|
|
|
33
|
1 |
|
if (preg_match('/^\s*User-Agent\: (.*)$/i', $line, $matches)) { |
|
34
|
1 |
|
if ($ua !== true) { |
|
35
|
1 |
|
$this->populateRules($rules, $current); |
|
36
|
1 |
|
$current = []; |
|
37
|
|
|
} |
|
38
|
1 |
|
$current[] = new Rule($matches[1]); |
|
39
|
1 |
|
$ua = true; |
|
40
|
|
|
} else { |
|
41
|
1 |
|
$ua = false; |
|
42
|
1 |
|
$this->parseLine($current, $line, $rules); |
|
43
|
|
|
} |
|
44
|
|
|
} |
|
45
|
1 |
|
$this->populateRules($rules, $current); |
|
46
|
1 |
|
return $rules; |
|
47
|
|
|
} |
|
48
|
|
|
|
|
49
|
|
|
/** |
|
50
|
|
|
* Handle content to build a valid instance |
|
51
|
|
|
* @param string|Content $content |
|
52
|
|
|
* @return Content |
|
53
|
|
|
*/ |
|
54
|
|
|
private function handleContent($content) |
|
55
|
|
|
{ |
|
56
|
1 |
|
if (is_string($content)) { |
|
57
|
1 |
|
$content = new Content($content); |
|
58
|
|
|
} |
|
59
|
1 |
|
if (!($content instanceof Content)) { |
|
60
|
1 |
|
throw (new InvalidContentException( |
|
61
|
1 |
|
'Content must be a `string` or a `Content` instance' |
|
62
|
1 |
|
))->setContent($content); |
|
63
|
|
|
} |
|
64
|
|
|
|
|
65
|
1 |
|
return $content; |
|
66
|
|
|
} |
|
67
|
|
|
|
|
68
|
|
|
/** |
|
69
|
|
|
* Transform file content to structured Rules |
|
70
|
|
|
* @param string|Content $content |
|
71
|
|
|
* @return Rules |
|
72
|
|
|
*/ |
|
73
|
|
|
public static function parse($content) |
|
74
|
|
|
{ |
|
75
|
1 |
|
$parser = new self(); |
|
76
|
1 |
|
return $parser->analyze($content); |
|
77
|
|
|
} |
|
78
|
|
|
|
|
79
|
|
|
/** |
|
80
|
|
|
* Parse a line of data |
|
81
|
|
|
* @param array &$current |
|
82
|
|
|
* @param string $line |
|
83
|
|
|
*/ |
|
84
|
|
|
private function parseLine(array &$current, $line, Rules $rules) |
|
85
|
|
|
{ |
|
86
|
1 |
|
if (preg_match('/^\s*(Allow|Disallow): ((\*).+|(\/.*))$/i', $line, $matches)) { |
|
87
|
1 |
|
$match = array_values( |
|
88
|
|
|
array_filter( |
|
89
|
1 |
|
array_slice($matches, 3) |
|
90
|
|
|
) |
|
91
|
|
|
); |
|
92
|
|
|
|
|
93
|
1 |
|
$this->apply( |
|
94
|
|
|
$current, |
|
95
|
1 |
|
strtolower($matches[1]), |
|
96
|
1 |
|
$match[0] |
|
97
|
|
|
); |
|
98
|
1 |
|
} elseif (preg_match('/^\s*Sitemap: (.*)$/i', $line, $matches)) { |
|
99
|
1 |
|
$rules->addSitemap($matches[1]); |
|
100
|
|
|
} |
|
101
|
1 |
|
} |
|
102
|
|
|
|
|
103
|
|
|
/** |
|
104
|
|
|
* Apply a method on all element of a given array |
|
105
|
|
|
* @param array $data |
|
106
|
|
|
* @param string $method |
|
107
|
|
|
* @param string $param |
|
108
|
|
|
*/ |
|
109
|
|
|
private function apply(array $data, $method, $param) |
|
110
|
|
|
{ |
|
111
|
1 |
|
array_walk($data, function (Rule $item) use ($method, $param) { |
|
112
|
1 |
|
$item->$method($param); |
|
113
|
1 |
|
}); |
|
114
|
1 |
|
} |
|
115
|
|
|
|
|
116
|
|
|
/** |
|
117
|
|
|
* Populate rules property with build Rule instance |
|
118
|
|
|
* @param Rules $rules |
|
119
|
|
|
* @param array $current Collection of Rule objects |
|
120
|
|
|
*/ |
|
121
|
|
|
private function populateRules(Rules $rules, array $current) |
|
122
|
|
|
{ |
|
123
|
1 |
|
foreach ($current as $item) { |
|
124
|
1 |
|
$rules->add($item); |
|
125
|
|
|
} |
|
126
|
1 |
|
} |
|
127
|
|
|
} |
|
128
|
|
|
|
bee4 /
robots.txt
