@@ -292,6 +292,9 @@ discard block |
||
292 | 292 | strlen($haystack); |
293 | 293 | } |
294 | 294 | |
295 | + /** |
|
296 | + * @param integer $readMaximumBytes |
|
297 | + */ |
|
295 | 298 | protected function convertBodyToString(StreamInterface $bodyStream, $readMaximumBytes = 1024 * 1024 * 2): string |
296 | 299 | { |
297 | 300 | $bodyStream->rewind(); |
@@ -302,7 +305,7 @@ discard block |
||
302 | 305 | } |
303 | 306 | |
304 | 307 | /** |
305 | - * @param ResponseInterface|null $response |
|
308 | + * @param ResponseInterface $response |
|
306 | 309 | * @param CrawlUrl $crawlUrl |
307 | 310 | */ |
308 | 311 | protected function handleCrawled(ResponseInterface $response, CrawlUrl $crawlUrl) |
@@ -438,6 +441,9 @@ discard block |
||
438 | 441 | return in_array($uri->getScheme(), ['http', 'https']); |
439 | 442 | } |
440 | 443 | |
444 | + /** |
|
445 | + * @return Node |
|
446 | + */ |
|
441 | 447 | protected function addtoDepthTree(Node $node, UriInterface $url, UriInterface $parentUrl) |
442 | 448 | { |
443 | 449 | $returnNode = null; |
@@ -3,25 +3,25 @@ |
||
3 | 3 | namespace Spatie\Crawler; |
4 | 4 | |
5 | 5 | use Generator; |
6 | -use Tree\Node\Node; |
|
7 | -use GuzzleHttp\Pool; |
|
8 | 6 | use GuzzleHttp\Client; |
9 | -use GuzzleHttp\Psr7\Uri; |
|
7 | +use GuzzleHttp\Exception\RequestException; |
|
8 | +use GuzzleHttp\Pool; |
|
10 | 9 | use GuzzleHttp\Psr7\Request; |
11 | -use Spatie\Robots\RobotsTxt; |
|
12 | -use InvalidArgumentException; |
|
13 | -use Spatie\Robots\RobotsMeta; |
|
10 | +use GuzzleHttp\Psr7\Uri; |
|
14 | 11 | use GuzzleHttp\RequestOptions; |
15 | -use Spatie\Robots\RobotsHeaders; |
|
12 | +use InvalidArgumentException; |
|
13 | +use Psr\Http\Message\ResponseInterface; |
|
14 | +use Psr\Http\Message\StreamInterface; |
|
16 | 15 | use Psr\Http\Message\UriInterface; |
17 | 16 | use Spatie\Browsershot\Browsershot; |
18 | -use Psr\Http\Message\StreamInterface; |
|
19 | -use Symfony\Component\DomCrawler\Link; |
|
20 | -use Psr\Http\Message\ResponseInterface; |
|
21 | -use Spatie\Crawler\CrawlQueue\CrawlQueue; |
|
22 | -use GuzzleHttp\Exception\RequestException; |
|
23 | 17 | use Spatie\Crawler\CrawlQueue\CollectionCrawlQueue; |
18 | +use Spatie\Crawler\CrawlQueue\CrawlQueue; |
|
19 | +use Spatie\Robots\RobotsHeaders; |
|
20 | +use Spatie\Robots\RobotsMeta; |
|
21 | +use Spatie\Robots\RobotsTxt; |
|
24 | 22 | use Symfony\Component\DomCrawler\Crawler as DomCrawler; |
23 | +use Symfony\Component\DomCrawler\Link; |
|
24 | +use Tree\Node\Node; |
|
25 | 25 | |
26 | 26 | class Crawler |
27 | 27 | { |