Crawler::getCacheFolder() - Code Metrics - Inspection of "init" - PiedWeb/SeoPocketCrawler - Measure and Improve Code Quality continuously with Scrutinizer

Completed
Push — master ( 7ba6fb...31ed5b )

by Dev
created 2019-01-15 17:39 UTC
Crawler::getCacheFolder() A

↳ Parent: Crawler
Complexity

Conditions	1
Paths	1
Size

Total Lines	3
Code Lines	1
Duplication

Lines	0
Ratio	0 %
Code Coverage

Tests	2
CRAP Score	1
Importance

Changes
Metric	Value
cc	1
eloc	1
nc	1
nop	0
dl	0
loc	3
ccs	2
cts	2
cp	1
crap	1
rs	10
c	0
b	0
f	0
<?php

namespace PiedWeb\SeoPocketCrawler;

use PiedWeb\UrlHarvester\Harvest;
use PiedWeb\UrlHarvester\Indexable;
use Spatie\Robots\RobotsTxt;

class Crawler
{
    protected $userAgent;
    protected $project;
    protected $ignore;
    protected $limit;

    protected $currentClick = 0;

    protected $counter = 0;

    protected $base;
    protected $urls = [];

    public function __construct(string $startUrl, string $ignore, int $limit, string $userAgent)
    {
        $this->urls[$startUrl] = null;
        $this->base = Harvest::getDomainAndSchemeFrom($startUrl);
        $this->project = preg_replace("([^\w\s\d\-_~,;\[\]\(\).])", '', $startUrl);
        $this->ignore = new RobotsTxt($ignore);
        $this->userAgent = $userAgent;
        $this->limit = $limit;

        $this->initRecorderAndCache();
    }

    public function getDataFolder()
    {
        return __DIR__.'/../data/'.$this->project;
    }

    public function getCacheFolder()
    {
        return __DIR__.'/../cache/'.$this->project;
    }

    protected function initRecorderAndCache()
    {
        $this->recorder = new Recorder($this->getDataFolder());


        exec('rm -rf '.$this->getDataFolder());
        exec('rm -rf '.$this->getCacheFolder());

        if (!file_exists($this->getDataFolder())) {
            mkdir($this->getDataFolder());
            mkdir($this->getDataFolder().'/links');
            mkdir($this->getCacheFolder());
        }
    }

    public function crawl(bool $debug = false)
    {
        $nothingUpdated = true;

        if ($debug) {
            echo PHP_EOL.PHP_EOL.'// -----'.PHP_EOL.'// '.$this->counter.' crawled / '
                        .count($this->urls).' found '.PHP_EOL.'// -----'.PHP_EOL;
        }

        foreach ($this->urls as $urlToParse => $url) {
            if (null !== $url && (false === $url->can_be_crawled || true === $url->can_be_crawled)) { // déjà crawlé
                continue;
            }

            if ($debug) {
                echo '    '.$urlToParse.PHP_EOL;
            }

            $nothingUpdated = false;
            ++$this->counter;

            $this->harvest($urlToParse);
        }

        ++$this->currentClick;

        $record = $nothingUpdated || $this->currentClick >= $this->limit;

        return $record ? $this->recorder->record($this->urls) : $this->crawl($debug);
class A
{
    function getObject()
    {
        return null;
    }

}

$a = new A();
if ($a->getObject()) {
    }

    protected function cache($harvest)
    {
        if (false === strpos($harvest->getResponse()->getContentType(), 'text/html')) {
            return;
        }

        $url = ltrim($harvest->getAbsoluteInternalLink($harvest->getResponse()->getEffectiveUrl()), '/');
        $urlPart = explode('/', $url);
        $folder = $this->getCacheFolder();

        for ($i = 0; $i < count($urlPart); ++$i) {
for ($i=0; $i<count($array); $i++) { // calls count() on each iteration
}

// Better
for ($i=0, $c=count($array); $i<$c; $i++) { // calls count() just once
}
            if ($i == count($urlPart) - 1) {
                $filename = empty($urlPart[$i]) ? 'index.html' : $urlPart[$i];
                @file_put_contents($folder.'/'.$filename, $harvest->getResponse()->getContent());
                /** @scrutinizer ignore-unhandled */ @file_put_contents($folder.'/'.$filename, $harvest->getResponse()->getContent());
            } else {
                $folder .= '/'.$urlPart[$i];
                if (!file_exists($folder)) {
                    @mkdir($folder);
                    /** @scrutinizer ignore-unhandled */ @mkdir($folder);
                }
            }
        }
    }

    protected function harvest(string $urlToParse)
    {
        $url = $this->urls[$urlToParse] = $this->urls[$urlToParse] ?? new Url($urlToParse, $this->currentClick);

        $url->updated_at = date('Ymd');
        $url->can_be_crawled = $this->ignore->allows($urlToParse, $this->userAgent);

        if (false === $url->can_be_crawled) {
            return;
        }

        $harvest = Harvest::fromUrl($urlToParse, $this->userAgent);

        if (!$harvest instanceof Harvest) {
            $url->indexable = Indexable::NOT_INDEXABLE_NETWORK_ERROR;

            return;
        }

        $url->indexable = $harvest->isIndexable();

        if (Indexable::NOT_INDEXABLE_3XX === $url->indexable) {
            if ($redir = false !== $harvest->getRedirection()) {
                $links = Harvest::LINK_INTERNAL === $harvest->getType($redir) ? [$redir] : [];
                $links = Harvest::LINK_INTERNAL === $harvest->getType(/** @scrutinizer ignore-type */ $redir) ? [$redir] : [];
            }
        } else {
            $this->cache($harvest);

            $this->recorder->recordOutboundLink($url, $harvest->getLinks());

            $url->links = count($harvest->getLinks());
            $url->links_duplicate = $harvest->getNbrDuplicateLinks();
            $url->links_internal = count($harvest->getLinks(Harvest::LINK_INTERNAL));
            $url->links_self = count($harvest->getLinks(Harvest::LINK_SELF));
            $url->links_sub = count($harvest->getLinks(Harvest::LINK_SUB));
            $url->links_external = count($harvest->getLinks(Harvest::LINK_EXTERNAL));

            $url->ratio_text_code = $harvest->getRatioTxtCode();
            $url->load_time = $harvest->getResponse()->getInfo('total_time');
            $url->size = $harvest->getResponse()->getInfo('size_download');

            $breadcrumb = $harvest->getBreadCrumb();
            if (is_array($breadcrumb)) {
                $url->breadcrumb_level = count($breadcrumb);

                $url->breadcrumb_fisrt = isset($breadcrumb[1]) ? $breadcrumb[1]->getCleanName() : '';

                $url->breadcrumb_text = $harvest->getBreadCrumb('//');

            }

            $url->title = $harvest->getUniqueTag('head title') ?? '';
            $url->kws = ','.implode(',', $harvest->getKws()).',';
            $url->h1 = $harvest->getUniqueTag('h1') ?? '';
        }

        foreach ($harvest->getLinks(Harvest::LINK_INTERNAL) as $link) {
            $linkUrl = $link->getPageUrl();
            $this->urls[$linkUrl] = $this->urls[$linkUrl] ?? new Url($linkUrl, ($this->currentClick + 1));
            $this->recorder->recordInboundLink($url, $this->urls[$linkUrl]);
            ++$this->urls[$linkUrl]->inboundlinks;
        }
    }

    public function recordLink(Url $from, To $url)
filter:
    dependency_paths: ["lib/*"]
    {
        file_put_contents($this->getDataFolder().'/links/To_'.$url->sha1.'.txt', $from->uri, FILE_APPEND);
    }
}

PiedWeb / SeoPocketCrawler

Push — master ( 7ba6fb...31ed5b )

Crawler::getCacheFolder() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like
