| 1 |  |  | <?php | 
            
                                                                                                            
                            
            
                                    
            
            
                | 2 |  |  | namespace vipnytt\RobotsTxtParser; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 3 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 4 |  |  | use Composer\CaBundle\CaBundle; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 5 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 6 |  |  | /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 7 |  |  |  * Class UriClient | 
            
                                                                                                            
                            
            
                                    
            
            
                | 8 |  |  |  * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 9 |  |  |  * @package vipnytt\RobotsTxtParser | 
            
                                                                                                            
                            
            
                                    
            
            
                | 10 |  |  |  */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 11 |  |  | class UriClient extends TxtClient | 
            
                                                                                                            
                            
            
                                    
            
            
                | 12 |  |  | { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 13 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 14 |  |  |      * User-agent | 
            
                                                                                                            
                            
            
                                    
            
            
                | 15 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 16 |  |  |     const CURL_USER_AGENT = 'RobotsTxtParser-VIPnytt/2.0 (+https://github.com/VIPnytt/RobotsTxtParser/blob/master/README.md)'; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 17 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 18 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 19 |  |  |      * Base uri | 
            
                                                                                                            
                            
            
                                    
            
            
                | 20 |  |  |      * @var string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 21 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 22 |  |  |     private $base; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 23 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 24 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 25 |  |  |      * Header parser | 
            
                                                                                                            
                            
            
                                    
            
            
                | 26 |  |  |      * @var Parser\HeaderParser | 
            
                                                                                                            
                            
            
                                    
            
            
                | 27 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 28 |  |  |     private $headerParser; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 29 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 30 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 31 |  |  |      * RequestClient timestamp | 
            
                                                                                                            
                            
            
                                    
            
            
                | 32 |  |  |      * @var int | 
            
                                                                                                            
                            
            
                                    
            
            
                | 33 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 34 |  |  |     private $time; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 35 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 36 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 37 |  |  |      * Status code | 
            
                                                                                                            
                            
            
                                    
            
            
                | 38 |  |  |      * @var int|null | 
            
                                                                                                            
                            
            
                                    
            
            
                | 39 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 40 |  |  |     private $statusCode; | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 41 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 42 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 43 |  |  |      * Effective uri | 
            
                                                                                                            
                            
            
                                    
            
            
                | 44 |  |  |      * @var string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 45 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 46 |  |  |     private $effectiveUri; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 47 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 48 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 49 |  |  |      * Cache-Control max-age | 
            
                                                                                                            
                            
            
                                    
            
            
                | 50 |  |  |      * @var int | 
            
                                                                                                            
                            
            
                                    
            
            
                | 51 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 52 |  |  |     private $maxAge; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 53 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 54 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 55 |  |  |      * Robots.txt contents | 
            
                                                                                                            
                            
            
                                    
            
            
                | 56 |  |  |      * @var string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 57 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 58 |  |  |     private $contents; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 59 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 60 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 61 |  |  |      * Robots.txt character encoding | 
            
                                                                                                            
                            
            
                                    
            
            
                | 62 |  |  |      * @var string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 63 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 64 |  |  |     private $encoding; | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 65 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 66 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 67 |  |  |      * RequestClient constructor. | 
            
                                                                                                            
                            
            
                                    
            
            
                | 68 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 69 |  |  |      * @param string $baseUri | 
            
                                                                                                            
                            
            
                                    
            
            
                | 70 |  |  |      * @param array $curlOptions | 
            
                                                                                                            
                            
            
                                    
            
            
                | 71 |  |  |      * @param int|null $byteLimit | 
            
                                                                                                            
                            
            
                                    
            
            
                | 72 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 73 |  |  |     public function __construct($baseUri, array $curlOptions = [], $byteLimit = self::BYTE_LIMIT) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 74 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 75 |  |  |         $this->base = $this->urlBase($this->urlEncode($baseUri)); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 76 |  |  |         if ($this->request($curlOptions) === false) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 77 |  |  |             $this->time = time(); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 78 |  |  |             $this->effectiveUri = $this->base . self::PATH; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 79 |  |  |             $this->statusCode = null; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 80 |  |  |             $this->contents = ''; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 81 |  |  |             $this->encoding = self::ENCODING; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 82 |  |  |             $this->maxAge = 0; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 83 |  |  |         } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 84 |  |  |         parent::__construct($this->base, $this->statusCode, $this->contents, $this->encoding, $this->effectiveUri, $byteLimit); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 85 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 86 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 87 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 88 |  |  |      * cURL request | 
            
                                                                                                            
                            
            
                                    
            
            
                | 89 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 90 |  |  |      * @param array $options | 
            
                                                                                                            
                            
            
                                    
            
            
                | 91 |  |  |      * @return bool | 
            
                                                                                                            
                            
            
                                    
            
            
                | 92 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 93 |  |  |     private function request($options = []) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 94 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 95 |  |  |         $this->headerParser = new Parser\HeaderParser(); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 96 |  |  |         $ch = curl_init(); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 97 |  |  |         curl_setopt_array($ch, [ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 98 |  |  |             CURLOPT_AUTOREFERER => true, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 99 |  |  |             CURLOPT_CAINFO => CaBundle::getSystemCaRootBundlePath(), | 
            
                                                                                                            
                            
            
                                    
            
            
                | 100 |  |  |             CURLOPT_CONNECTTIMEOUT => 30, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 101 |  |  |             CURLOPT_ENCODING => 'identity', | 
            
                                                                                                            
                            
            
                                    
            
            
                | 102 |  |  |             CURLOPT_FAILONERROR => false, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 103 |  |  |             CURLOPT_FOLLOWLOCATION => true, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 104 |  |  |             CURLOPT_FTPSSLAUTH => CURLFTPAUTH_DEFAULT, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 105 |  |  |             CURLOPT_HEADER => false, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 106 |  |  |             CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_NONE, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 107 |  |  |             CURLOPT_IPRESOLVE => CURL_IPRESOLVE_WHATEVER, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 108 |  |  |             CURLOPT_NOBODY => false, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 109 |  |  |             CURLOPT_MAXREDIRS => self::MAX_REDIRECTS, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 110 |  |  |             CURLOPT_RETURNTRANSFER => true, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 111 |  |  |             CURLOPT_SSL_VERIFYHOST => 2, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 112 |  |  |             CURLOPT_SSL_VERIFYPEER => true, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 113 |  |  |             //CURLOPT_SSL_VERIFYSTATUS => true, // PHP 7.0.7 | 
            
                                                                                                            
                            
            
                                    
            
            
                | 114 |  |  |             CURLOPT_TIMEOUT => 120, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 115 |  |  |             CURLOPT_USERAGENT => self::CURL_USER_AGENT, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 116 |  |  |             CURLOPT_USERPWD => 'anonymous:', | 
            
                                                                                                            
                            
            
                                    
            
            
                | 117 |  |  |         ]); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 118 |  |  |         curl_setopt_array($ch, $options); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 119 |  |  |         curl_setopt($ch, CURLOPT_HEADERFUNCTION, [$this->headerParser, 'curlCallback']); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 120 |  |  |         curl_setopt($ch, CURLOPT_URL, $this->base . self::PATH); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 121 |  |  |         if (($this->contents = curl_exec($ch)) === false) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 122 |  |  |             return false; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 123 |  |  |         } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 124 |  |  |         $this->time = time(); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 125 |  |  |         $this->statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); // also works with FTP status codes | 
            
                                                                                                            
                            
            
                                    
            
            
                | 126 |  |  |         $this->effectiveUri = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 127 |  |  |         curl_close($ch); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 128 |  |  |         $this->encoding = $this->headerParser->getCharset(); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 129 |  |  |         $this->maxAge = $this->headerParser->getMaxAge(); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 130 |  |  |         return true; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 131 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 132 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 133 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 134 |  |  |      * Base uri | 
            
                                                                                                            
                            
            
                                    
            
            
                | 135 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 136 |  |  |      * @return string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 137 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 138 |  |  |     public function getBaseUri() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 139 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 140 |  |  |         return $this->base; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 141 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 142 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 143 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 144 |  |  |      * Effective uri | 
            
                                                                                                            
                            
            
                                    
            
            
                | 145 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 146 |  |  |      * @return string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 147 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 148 |  |  |     public function getEffectiveUri() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 149 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 150 |  |  |         return $this->effectiveUri; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 151 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 152 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 153 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 154 |  |  |      * Status code | 
            
                                                                                                            
                            
            
                                    
            
            
                | 155 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 156 |  |  |      * @return int|null | 
            
                                                                                                            
                            
            
                                    
            
            
                | 157 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 158 |  |  |     public function getStatusCode() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 159 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 160 |  |  |         return $this->statusCode; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 161 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 162 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 163 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 164 |  |  |      * Body content | 
            
                                                                                                            
                            
            
                                    
            
            
                | 165 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 166 |  |  |      * @return string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 167 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 168 |  |  |     public function getContents() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 169 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 170 |  |  |         return $this->contents; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 171 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 172 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 173 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 174 |  |  |      * Encoding | 
            
                                                                                                            
                            
            
                                    
            
            
                | 175 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 176 |  |  |      * @return string | 
            
                                                                                                            
                            
            
                                    
            
            
                | 177 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 178 |  |  |     public function getEncoding() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 179 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 180 |  |  |         return $this->encoding; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 181 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 182 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 183 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 184 |  |  |      * Next update timestamp | 
            
                                                                                                            
                            
            
                                    
            
            
                | 185 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 186 |  |  |      * @return int | 
            
                                                                                                            
                            
            
                                    
            
            
                | 187 |  |  |      */ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 188 |  |  |     public function nextUpdate() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 189 |  |  |     { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 190 |  |  |         if ($this->statusCode === 503) { | 
            
                                                                                                            
                            
            
                                    
            
            
                | 191 |  |  |             return $this->time + min(self::CACHE_TIME, $this->headerParser->getRetryAfter($this->time)); | 
            
                                                                                                            
                            
            
                                    
            
            
                | 192 |  |  |         } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 193 |  |  |         return $this->time + self::CACHE_TIME; | 
            
                                                                                                            
                            
            
                                    
            
            
                | 194 |  |  |     } | 
            
                                                                                                            
                            
            
                                    
            
            
                | 195 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 196 |  |  |     /** | 
            
                                                                                                            
                            
            
                                    
            
            
                | 197 |  |  |      * Valid until timestamp | 
            
                                                                                                            
                            
            
                                    
            
            
                | 198 |  |  |      * | 
            
                                                                                                            
                            
            
                                    
            
            
                | 199 |  |  |      * @return int | 
            
                                                                                                            
                                                                
            
                                    
            
            
                | 200 |  |  |      */ | 
            
                                                        
            
                                    
            
            
                | 201 |  |  |     public function validUntil() | 
            
                                                        
            
                                    
            
            
                | 202 |  |  |     { | 
            
                                                        
            
                                    
            
            
                | 203 |  |  |         return $this->time + max(self::CACHE_TIME, $this->maxAge); | 
            
                                                        
            
                                    
            
            
                | 204 |  |  |     } | 
            
                                                        
            
                                    
            
            
                | 205 |  |  | } | 
            
                                                        
            
                                    
            
            
                | 206 |  |  |  |