PiedWeb / SeoPocketCrawler

src/Crawler.php

<?php

namespace PiedWeb\SeoPocketCrawler;

use PiedWeb\UrlHarvester\Harvest;
use PiedWeb\UrlHarvester\Indexable;
use Spatie\Robots\RobotsTxt;

class Crawler
{
    protected $userAgent;
    protected $project;
    protected $ignore;
    protected $limit;
    protected $recorder;

    protected $currentClick = 0;

    protected $counter = 0;

    protected $base;
    protected $urls = [];

    public function __construct(string $startUrl, string $ignore, int $limit, string $userAgent)
    {
        $this->urls[$startUrl] = null;
        $this->base = Harvest::getDomainAndSchemeFrom($startUrl);
        $this->project = preg_replace("([^\w\s\d\-_~,;\[\]\(\).])", '', $startUrl);
        $this->ignore = new RobotsTxt($ignore);
        $this->userAgent = $userAgent;
        $this->limit = $limit;

        $this->initRecorderAndCache();
    }

    public function getDataFolder()
    {
        return __DIR__.'/../data/'.$this->project;
    }

    public function getCacheFolder()
    {
        return __DIR__.'/../cache/'.$this->project;
    }

    protected function initRecorderAndCache()
    {
        $this->recorder = new Recorder($this->getDataFolder());

        exec('rm -rf '.$this->getDataFolder());
        exec('rm -rf '.$this->getCacheFolder());

        if (!file_exists($this->getDataFolder())) {
            mkdir($this->getDataFolder());
            mkdir($this->getDataFolder().'/links');
            mkdir($this->getCacheFolder());
        }
    }

    public function crawl(bool $debug = false)
    {
        $nothingUpdated = true;

        if ($debug) {
            echo PHP_EOL.PHP_EOL.'// -----'.PHP_EOL.'// '.$this->counter.' crawled / '
                        .count($this->urls).' found '.PHP_EOL.'// -----'.PHP_EOL;
        }

        foreach ($this->urls as $urlToParse => $url) {
            if (null !== $url && (false === $url->can_be_crawled || true === $url->can_be_crawled)) { // déjà crawlé
                continue;
            }

            if ($debug) {
                echo '    '.$urlToParse.PHP_EOL;
            }

            $nothingUpdated = false;
            ++$this->counter;

            $this->harvest($urlToParse);
        }

        ++$this->currentClick;

        $record = $nothingUpdated || $this->currentClick >= $this->limit;

        return $record ? $this->recorder->record($this->urls) : $this->crawl($debug);
    }

    protected function cache($harvest)
    {
        if (false === strpos($harvest->getResponse()->getContentType(), 'text/html')) {
            return;
        }

        $url = ltrim($harvest->getAbsoluteInternalLink($harvest->getResponse()->getEffectiveUrl()), '/');
        $urlPart = explode('/', $url);
        $folder = $this->getCacheFolder();

        $urlPartLenght = count($urlPart);
        for ($i = 0; $i < $urlPartLenght; ++$i) {
            if ($i == count($urlPart) - 1) {
                $filename = empty($urlPart[$i]) ? 'index.html' : $urlPart[$i];
                file_put_contents($folder.'/'.$filename, $harvest->getResponse()->getContent());
            } else {
                $folder .= '/'.$urlPart[$i];
                if (!file_exists($folder)) {
                    mkdir($folder);
                }
            }
        }
    }

    protected function harvest(string $urlToParse)
    {
        $url = $this->urls[$urlToParse] = $this->urls[$urlToParse] ?? new Url($urlToParse, $this->currentClick);

        $url->updated_at = date('Ymd');
        $url->can_be_crawled = $this->ignore->allows($urlToParse, $this->userAgent);

        if (false === $url->can_be_crawled) {
            return;
        }

        $harvest = Harvest::fromUrl($urlToParse, $this->userAgent);

        if (!$harvest instanceof Harvest) {
            $url->indexable = Indexable::NOT_INDEXABLE_NETWORK_ERROR;

            return;
        }

        $url->indexable = $harvest->isIndexable();

        if (Indexable::NOT_INDEXABLE_3XX === $url->indexable) {
            $redir = $harvest->getRedirection();
            if (false !== $redir) {
                $links = Harvest::LINK_INTERNAL === $harvest->getType($redir) ? [$redir] : [];

The condition PiedWeb\UrlHarvester\Harvest::LINK_INTERNAL === $harvest->getType($redir) is always false.

The assignment to $links is dead and can be removed.

            }
        } else {
            $this->cache($harvest);

            $this->recorder->recordOutboundLink($url, $harvest->getLinks());

            $url->links = count($harvest->getLinks());
            $url->links_duplicate = $harvest->getNbrDuplicateLinks();
            $url->links_internal = count($harvest->getLinks(Harvest::LINK_INTERNAL));
            $url->links_self = count($harvest->getLinks(Harvest::LINK_SELF));
            $url->links_sub = count($harvest->getLinks(Harvest::LINK_SUB));
            $url->links_external = count($harvest->getLinks(Harvest::LINK_EXTERNAL));

            $url->ratio_text_code = $harvest->getRatioTxtCode();
            $url->load_time = $harvest->getResponse()->getInfo('total_time');
            $url->size = $harvest->getResponse()->getInfo('size_download');

            $breadcrumb = $harvest->getBreadCrumb();
            if (is_array($breadcrumb)) {
                $url->breadcrumb_level = count($breadcrumb);
                $url->breadcrumb_fisrt = isset($breadcrumb[1]) ? $breadcrumb[1]->getCleanName() : '';

The property breadcrumb_fisrt does not exist on PiedWeb\SeoPocketCrawler\Url. Did you mean breadcrumb_first?

                $url->breadcrumb_text = $harvest->getBreadCrumb('//');
            }

            $url->title = $harvest->getUniqueTag('head title') ?? '';
            $url->kws = ','.implode(',', $harvest->getKws()).',';
            $url->h1 = $harvest->getUniqueTag('h1') ?? '';
        }

        foreach ($harvest->getLinks(Harvest::LINK_INTERNAL) as $link) {
            $linkUrl = $link->getPageUrl();
            $this->urls[$linkUrl] = $this->urls[$linkUrl] ?? new Url($linkUrl, ($this->currentClick + 1));
            $this->recorder->recordInboundLink($url, $this->urls[$linkUrl]);
            ++$this->urls[$linkUrl]->inboundlinks;
        }
    }
}