spatie / crawler

CrawlRealSite.php

<?php

For compatibility and reusability of your code, PSR1 recommends that a file should introduce either new symbols (like classes, functions, etc.) or have side-effects (like outputting something, or including other files), but not both at the same time. The first symbol is defined on line 10 and the first side effect is on line 8.

use Spatie\Crawler\Crawler;
use Spatie\Crawler\CrawlInternalUrls;
use Spatie\Crawler\CrawlObserver;
use Psr\Http\Message\UriInterface;

include "vendor/autoload.php";

$observer = new class implements CrawlObserver

PSR1 recommends that each class must be in a namespace of at least one level to avoid collisions.

{
    /**
     * Called when the crawler will crawl the url.
     *
     * @param \Psr\Http\Message\UriInterface   $url
     */
    public function willCrawl(UriInterface $url)
    {

    }

    /**
     * Called when the crawler has crawled the given url.
     *
     * @param \Psr\Http\Message\UriInterface $url
     * @param \Psr\Http\Message\ResponseInterface|null $response
     * @param \Psr\Http\Message\UriInterface|null $foundOnUrl
     */
    public function hasBeenCrawled(UriInterface $url, $response, ?UriInterface $foundOnUrl = null)
    {

    }

    /**
     * Called when the crawl has ended.
     */
    public function finishedCrawling()
    {

    }
};


Crawler::create()
    ->setCrawlObserver($observer)
    ->setCrawlProfile(new CrawlInternalUrls('https://spatie.be'))
    ->startCrawling('https://spatie.be');