for testing and deploying your application
for finding and fixing issues
for empowering human code reviews
<?php
namespace Spatie\Crawler\Handlers;
use Psr\Http\Message\ResponseInterface;
use Spatie\Crawler\CrawlerRobots;
use Spatie\Crawler\CrawlSubdomains;
class CrawlRequestFulfilled extends CrawlRequestFulfilledAbstract
{
public function __invoke(ResponseInterface $response, $index)
$robots = new CrawlerRobots($response, $this->crawler->mustRespectRobots());
if (! $robots->mayIndex()) {
return;
}
$crawlUrl = $this->crawler->getCrawlQueue()->getUrlById($index);
$this->handleCrawled($response, $crawlUrl);
if (! $this->crawler->getCrawlProfile() instanceof CrawlSubdomains) {
if ($crawlUrl->url->getHost() !== $this->crawler->getBaseUrl()->getHost()) {
if (! $robots->mayFollow()) {
$body = $this->convertBodyToString($response->getBody(), $this->crawler->getMaximumResponseSize());
$this->linkAdder->addFromHtml($body, $crawlUrl->url);
spatie /
crawler
