| 1 |  |  | <?php | 
            
                                                                                                            
                            
            
                                    
            
            
                | 2 |  |  | namespace vipnytt\RobotsTxtParser\Parser; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 3 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 4 |  |  | use vipnytt\RobotsTxtParser\Exceptions\ParserException; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 5 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 6 |  |  | /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 7 |  |  |  * Trait Toolbox | 
            
                                                                                                            
                            
            
                                    
            
            
                | 8 |  |  |  * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 9 |  |  |  * @package vipnytt\RobotsTxtParser\Parser | 
            
                                                                                                            
                                                                
            
                                    
            
            
                | 10 |  |  |  */ | 
            
                                                                        
                            
            
                                    
            
            
                | 11 |  |  | trait Toolbox | 
            
                                                                        
                            
            
                                    
            
            
                | 12 |  |  | { | 
            
                                                                        
                            
            
                                    
            
            
                | 13 |  |  |     /** | 
            
                                                                        
                            
            
                                    
            
            
                | 14 |  |  |      * Check basic rule | 
            
                                                                        
                            
            
                                    
            
            
                | 15 |  |  |      * | 
            
                                                                        
                            
            
                                    
            
            
                | 16 |  |  |      * @param string $path | 
            
                                                                        
                            
            
                                    
            
            
                | 17 |  |  |      * @param array $paths | 
            
                                                                        
                            
            
                                    
            
            
                | 18 |  |  |      * @return bool | 
            
                                                                        
                            
            
                                    
            
            
                | 19 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 20 |  |  |     protected function checkPath($path, $paths) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 21 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 22 |  |  |         foreach ($paths as $rule) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 23 |  |  |             $escape = ['?' => '\?', '.' => '\.', '*' => '.*']; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 24 |  |  |             foreach ($escape as $search => $replace) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 25 |  |  |                 /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 26 |  |  |                  * str_replace needs to be replaced by something better, multi-byte safe | 
            
                                                                                                            
                            
            
                                    
            
            
                | 27 |  |  |                  */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 28 |  |  |                 $rule = str_replace($search, $replace, $rule); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 29 |  |  |             } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 30 |  |  |             /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 31 |  |  |              * Warning: preg_match need to be replaced | 
            
                                                                                                            
                            
            
                                    
            
            
                | 32 |  |  |              * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 33 |  |  |              * Bug report | 
            
                                                                                                            
                            
            
                                    
            
            
                | 34 |  |  |              * @link https://github.com/t1gor/Robots.txt-Parser-Class/issues/62 | 
            
                                                                                                            
                            
            
                                    
            
            
                | 35 |  |  |              * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 36 |  |  |              * An robots.txt parser, where a bug-fix is planned | 
            
                                                                                                            
                            
            
                                    
            
            
                | 37 |  |  |              * @link https://github.com/diggin/Diggin_RobotRules | 
            
                                                                                                            
                            
            
                                    
            
            
                | 38 |  |  |              * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 39 |  |  |              * The solution? | 
            
                                                                                                            
                            
            
                                    
            
            
                | 40 |  |  |              * PHP PEG (parsing expression grammar) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 41 |  |  |              * @link https://github.com/hafriedlander/php-peg | 
            
                                                                                                            
                            
            
                                    
            
            
                | 42 |  |  |              */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 43 |  |  |             try { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 44 |  |  |                 if (preg_match('#' . $rule . '#', $path)) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 45 |  |  |                     if (mb_stripos($rule, '$') !== false) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 46 |  |  |                         if (mb_strlen($rule) - 1 == mb_strlen($path)) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 47 |  |  |                             return true; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 48 |  |  |                         } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 49 |  |  |                     } else { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 50 |  |  |                         return true; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 51 |  |  |                     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 52 |  |  |                 } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 53 |  |  |             } catch (\Exception $e) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 54 |  |  |                 // An preg_match bug has occurred | 
            
                                                                                                            
                            
            
                                    
            
            
                | 55 |  |  |             } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 56 |  |  |         } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 57 |  |  |         return false; | 
            
                                                                                                            
                                                                
            
                                    
            
            
                | 58 |  |  |     } | 
            
                                                                        
                            
            
                                    
            
            
                | 59 |  |  |  | 
            
                                                                        
                            
            
                                    
            
            
                | 60 |  |  |     /** | 
            
                                                                        
                            
            
                                    
            
            
                | 61 |  |  |      * Generate directive/rule pair | 
            
                                                                        
                            
            
                                    
            
            
                | 62 |  |  |      * | 
            
                                                                        
                            
            
                                    
            
            
                | 63 |  |  |      * @param string $line | 
            
                                                                        
                            
            
                                    
            
            
                | 64 |  |  |      * @param array $whiteList | 
            
                                                                        
                            
            
                                    
            
            
                | 65 |  |  |      * @return array|false | 
            
                                                                        
                            
            
                                    
            
            
                | 66 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 67 |  |  |     protected function generateRulePair($line, $whiteList) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 68 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 69 |  |  |         $whiteList = array_map('mb_strtolower', $whiteList); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 70 |  |  |         // Split by directive and rule | 
            
                                                                                                            
                            
            
                                    
            
            
                | 71 |  |  |         $pair = array_map('trim', mb_split(':', $line, 2)); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 72 |  |  |         // Check if the line contains a rule | 
            
                                                                                                            
                            
            
                                    
            
            
                | 73 |  |  |         if ( | 
            
                                                                                                            
                            
            
                                    
            
            
                | 74 |  |  |             empty($pair[1]) || | 
            
                                                                                                            
                            
            
                                    
            
            
                | 75 |  |  |             empty($pair[0]) || | 
            
                                                                                                            
                            
            
                                    
            
            
                | 76 |  |  |             !in_array(($pair[0] = mb_strtolower($pair[0])), $whiteList) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 77 |  |  |         ) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 78 |  |  |             // Line does not contain any supported directive | 
            
                                                                                                            
                            
            
                                    
            
            
                | 79 |  |  |             return false; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 80 |  |  |         } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 81 |  |  |         return [ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 82 |  |  |             'directive' => $pair[0], | 
            
                                                                                                            
                            
            
                                    
            
            
                | 83 |  |  |             'value' => $pair[1], | 
            
                                                                                                            
                            
            
                                    
            
            
                | 84 |  |  |         ]; | 
            
                                                                                                            
                                                                
            
                                    
            
            
                | 85 |  |  |     } | 
            
                                                                        
                            
            
                                    
            
            
                | 86 |  |  |  | 
            
                                                                        
                            
            
                                    
            
            
                | 87 |  |  |     /** | 
            
                                                                        
                            
            
                                    
            
            
                | 88 |  |  |      * Validate directive | 
            
                                                                        
                            
            
                                    
            
            
                | 89 |  |  |      * | 
            
                                                                        
                            
            
                                    
            
            
                | 90 |  |  |      * @param $directive | 
            
                                                                        
                            
            
                                    
            
            
                | 91 |  |  |      * @param $directives | 
            
                                                                        
                            
            
                                    
            
            
                | 92 |  |  |      * @return string | 
            
                                                                        
                            
            
                                    
            
            
                | 93 |  |  |      * @throws ParserException | 
            
                                                                        
                            
            
                                    
            
            
                | 94 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 95 |  |  |     protected function validateDirective($directive, $directives) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 96 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 97 |  |  |         if (!in_array($directive, $directives, true)) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 98 |  |  |             throw new ParserException('Directive is not allowed here'); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 99 |  |  |         } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 100 |  |  |         return mb_strtolower($directive); | 
            
                                                                                                            
                                                                
            
                                    
            
            
                | 101 |  |  |     } | 
            
                                                        
            
                                    
            
            
                | 102 |  |  | } | 
            
                                                        
            
                                    
            
            
                | 103 |  |  |  |