View GitHub Repository crawler

Schedule Inspection
Subscribe
Completed
Pull Request — master (#168)
by
unknown
01:49 queued 11s
created

CrawlRequestFulfilled::__construct()   A

↳ Parent: CrawlRequestFulfilled::__invoke()

Complexity

Conditions 1
Paths 1

Size

Total Lines 6
Code Lines 3

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
dl 0
loc 6
rs 9.4285
c 0
b 0
f 0
cc 1
eloc 3
nc 1
nop 1
1 
<?php
2 

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    3
                

                                    
                                                    
                
                
                
namespace Spatie\Crawler\Handlers;

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    4
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    5
                

                                    
                                                    
                
                
                
use Psr\Http\Message\ResponseInterface;

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    6
                

                                    
                                                    
                
                
                
use Spatie\Crawler\CrawlerRobots;

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    7
                

                                    
                                                    
                
                
                
use Spatie\Crawler\CrawlSubdomains;

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    8
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    9
                

                                    
                                                    
                
                
                
class CrawlRequestFulfilled extends CrawlRequestFulfilledAbstract

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    10
                

                                    
                                                    
                
                
                
{

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    11
                

                                    
                                                    
                
                
                
    public function __invoke(ResponseInterface $response, $index)

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    12
                

                                    
                                                    
                
                
                
    {

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    13
                

                                    
                                                    
                
                
                
        $robots = new CrawlerRobots($response, $this->crawler->mustRespectRobots());

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    14
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    15
                

                                    
                                                    
                
                
                
        if (! $robots->mayIndex()) {

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    16
                

                                    
                                                    
                
                
                
            return;

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    17
                

                                    
                                                    
                
                
                
        }

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    18
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    19
                

                                    
                                                    
                
                
                
        $crawlUrl = $this->crawler->getCrawlQueue()->getUrlById($index);

            


            
                                                                                                            
                                                                
            
                                    
            
            

                
                    20
                

                                    
                                                    
                
                
                

            


            
                                                                        
                            
            
                                    
            
            

                
                    21
                

                                    
                                                    
                
                
                
        $this->handleCrawled($response, $crawlUrl);

            


            
                                                                        
                            
            
                                    
            
            

                
                    22
                

                                    
                                                    
                
                
                

            


            
                                                                        
                            
            
                                    
            
            

                
                    23
                

                                    
                                                    
                
                
                
        if (! $this->crawler->getCrawlProfile() instanceof CrawlSubdomains) {

            


            
                                                                        
                            
            
                                    
            
            

                
                    24
                

                                    
                                                    
                
                
                
            if ($crawlUrl->url->getHost() !== $this->crawler->getBaseUrl()->getHost()) {

            


            
                                                                        
                            
            
                                    
            
            

                
                    25
                

                                    
                                                    
                
                
                
                return;

            


            
                                                                        
                            
            
                                    
            
            

                
                    26
                

                                    
                                                    
                
                
                
            }

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    27
                

                                    
                                                    
                
                
                
        }

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    28
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    29
                

                                    
                                                    
                
                
                
        if (! $robots->mayFollow()) {

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    30
                

                                    
                                                    
                
                
                
            return;

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    31
                

                                    
                                                    
                
                
                
        }

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    32
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    33
                

                                    
                                                    
                
                
                
        $body = $this->convertBodyToString($response->getBody(), $this->crawler->getMaximumResponseSize());

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    34
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    35
                

                                    
                                                    
                
                
                
        $this->linkAdder->addFromHtml($body, $crawlUrl->url);

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    36
                

                                    
                                                    
                
                
                
    }

            


            
                                                                                                            
                                                                
            
                                    
            
            

                
                    37
                

                                    
                                                    
                
                
                
}

            


            
                                                        
            
                                    
            
            

                
                    38