Crawler::loadRobotsTxt() - Code Metrics - Inspection of "init" - PiedWeb/SeoPocketCrawler - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( edfe10...b5c69a )

by Dev

created 2019-01-16 10:24 UTC

Crawler::loadRobotsTxt() A

↳ Parent: Crawler

Complexity

Conditions	2
Paths	2

Size

Total Lines	7
Code Lines	3

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	3
CRAP Score	2.0625

Importance

Changes

Metric	Value
cc	2
eloc	3
nc	2
nop	1
dl	0
loc	7
ccs	3
cts	4
cp	0.75
crap	2.0625
rs	10
c	0
b	0
f	0

<?php

namespace PiedWeb\SeoPocketCrawler;

use PiedWeb\UrlHarvester\Harvest;
use PiedWeb\UrlHarvester\Indexable;
use Spatie\Robots\RobotsTxt;

class Crawler
{
    protected $userAgent;
    protected $project;
    protected $ignore;
    protected $limit;
    protected $recorder;
    protected $robotsTxt;
    protected $wait = 0;

    protected $currentClick = 0;

    protected $counter = 0;

    protected $urls = [];

    public function __construct(string $startUrl, string $ignore, int $limit, string $userAgent)
    {
        $this->urls[$startUrl] = null;
        $this->project = preg_replace("([^\w\s\d\-_~,;\[\]\(\).])", '', $startUrl).'-'.date('ymd-Hi');
        $this->ignore = new RobotsTxt($ignore);
        $this->userAgent = $userAgent;
        $this->limit = $limit;

        $this->initRecorderAndCache();
    }

    public function getDataFolder()
    {
        return __DIR__.'/../data/'.$this->project;
    }

    public function getCacheFolder()
    {
        return __DIR__.'/../cache/'.$this->project;
    }

    protected function initRecorderAndCache()
    {
        $this->recorder = new Recorder($this->getDataFolder());

        exec('rm -rf '.$this->getDataFolder());
        exec('rm -rf '.$this->getCacheFolder());

        if (!file_exists($this->getDataFolder())) {
            mkdir($this->getDataFolder());
            mkdir($this->getDataFolder().'/links');
            mkdir($this->getCacheFolder());
        }
    }

    public function setWaitBetweenRequest(int $microSeconds = 100)
    {
        $this->wai = $microSeconds;

    }

    public function crawl(bool $debug = false)
    {
        $nothingUpdated = true;

        if ($debug) {
            echo PHP_EOL.PHP_EOL.'// -----'.PHP_EOL.'// '.$this->counter.' crawled / '
                        .count($this->urls).' found '.PHP_EOL.'// -----'.PHP_EOL;
        }

        foreach ($this->urls as $urlToParse => $url) {
            if (null !== $url && (false === $url->can_be_crawled || true === $url->can_be_crawled)) { // déjà crawlé
                continue;
            }

            if ($debug) {
                echo '    '.$urlToParse.PHP_EOL;
            }

            $nothingUpdated = false;
            ++$this->counter;

            $harvest = $this->harvest($urlToParse);

            $this->cacheRobotsTxt($harvest);
            usleep($this->wait);
        }

        ++$this->currentClick;

        $record = $nothingUpdated || $this->currentClick >= $this->limit;

        return $record ? $this->recorder->record($this->urls) : $this->crawl($debug);
    }

    protected function cacheRobotsTxt($harvest)
    {
        if (null === $this->robotsTxt && $harvest instanceof Harvest) {
            $this->robotsTxt = $harvest->getRobotsTxt();
        }

        return $this;
    }

    protected function loadRobotsTxt(Harvest $harvest)
    {
        if (null !== $this->robotsTxt) {
            $harvest->setRobotsTxt($this->robotsTxt);
        }

        return $this;
    }

    protected function cache($harvest)
    {
        if (false === strpos($harvest->getResponse()->getContentType(), 'text/html')) {
            return;
        }

        $url = ltrim($harvest->getAbsoluteInternalLink($harvest->getResponse()->getEffectiveUrl()), '/');
        $urlPart = explode('/', $url);
        $folder = $this->getCacheFolder();

        $urlPartLenght = count($urlPart);
        for ($i = 0; $i < $urlPartLenght; ++$i) {
            if ($i == count($urlPart) - 1) {
                $filename = empty($urlPart[$i]) ? 'index.html' : $urlPart[$i];
                file_put_contents($folder.'/'.$filename, $harvest->getResponse()->getContent());
            } else {
                $folder .= '/'.$urlPart[$i];
                if (!file_exists($folder)) {
                    mkdir($folder);
                }
            }
        }
    }

    protected function harvest(string $urlToParse)
    {
        $url = $this->urls[$urlToParse] = $this->urls[$urlToParse] ?? new Url($urlToParse, $this->currentClick);

        $url->updated_at = date('Ymd');
        $url->can_be_crawled = $this->ignore->allows($urlToParse, $this->userAgent);

        if (false === $url->can_be_crawled) {
            return;
        }

        $harvest = Harvest::fromUrl($urlToParse, $this->userAgent);

        if (!$harvest instanceof Harvest) {
            $url->indexable = Indexable::NOT_INDEXABLE_NETWORK_ERROR;

            return;
        }

        $this->loadRobotsTxt($harvest);

        $url->indexable = $harvest->isIndexable();

        if (Indexable::NOT_INDEXABLE_3XX === $url->indexable) {
            $redir = $harvest->getRedirection();
            if (false !== $redir) {
                $links = Harvest::LINK_INTERNAL === $harvest->getType($redir) ? [$redir] : [];

            }
        } else {
            $this->cache($harvest);

            $this->recorder->recordOutboundLink($url, $harvest->getLinks());

            $url->links = count($harvest->getLinks());
            $url->links_duplicate = $harvest->getNbrDuplicateLinks();
            $url->links_internal = count($harvest->getLinks(Harvest::LINK_INTERNAL));
            $url->links_self = count($harvest->getLinks(Harvest::LINK_SELF));
            $url->links_sub = count($harvest->getLinks(Harvest::LINK_SUB));
            $url->links_external = count($harvest->getLinks(Harvest::LINK_EXTERNAL));
            $links = $harvest->getLinks(Harvest::LINK_INTERNAL);

            $url->ratio_text_code = $harvest->getRatioTxtCode();
            $url->load_time = $harvest->getResponse()->getInfo('total_time');
            $url->size = $harvest->getResponse()->getInfo('size_download');

            $breadcrumb = $harvest->getBreadCrumb();
            if (is_array($breadcrumb)) {
                $url->breadcrumb_level = count($breadcrumb);
                $url->breadcrumb_first = isset($breadcrumb[1]) ? $breadcrumb[1]->getCleanName() : '';
                $url->breadcrumb_text = $harvest->getBreadCrumb('//');
            }

            $url->title = $harvest->getUniqueTag('head title') ?? '';
            $url->kws = ','.implode(',', $harvest->getKws()).',';
            $url->h1 = $harvest->getUniqueTag('h1') ?? '';
        }

        if (isset($links)) {
            foreach ($links as $link) {
                $linkUrl = $link->getPageUrl();
                $this->urls[$linkUrl] = $this->urls[$linkUrl] ?? new Url($linkUrl, ($this->currentClick + 1));
                $this->recorder->recordInboundLink($url, $this->urls[$linkUrl]);
                ++$this->urls[$linkUrl]->inboundlinks;
            }
        }

        return $harvest;
    }
}


1		<?php
2
3		namespace PiedWeb\SeoPocketCrawler;
4
5		use PiedWeb\UrlHarvester\Harvest;
6		use PiedWeb\UrlHarvester\Indexable;
7		use Spatie\Robots\RobotsTxt;
8
9		class Crawler
10		{
11		protected $userAgent;
12		protected $project;
13		protected $ignore;
14		protected $limit;
15		protected $recorder;
16		protected $robotsTxt;
17		protected $wait = 0;
18
19		protected $currentClick = 0;
20
21		protected $counter = 0;
22
23		protected $urls = [];
24
25	3	public function __construct(string $startUrl, string $ignore, int $limit, string $userAgent)
26		{
27	3	$this->urls[$startUrl] = null;
28	3	$this->project = preg_replace("([^\w\s\d\-_~,;\[\]\(\).])", '', $startUrl).'-'.date('ymd-Hi');
29	3	$this->ignore = new RobotsTxt($ignore);
30	3	$this->userAgent = $userAgent;
31	3	$this->limit = $limit;
32
33	3	$this->initRecorderAndCache();
34	3	}
35
36	3	public function getDataFolder()
37		{
38	3	return __DIR__.'/../data/'.$this->project;
39		}
40
41	3	public function getCacheFolder()
42		{
43	3	return __DIR__.'/../cache/'.$this->project;
44		}
45
46	3	protected function initRecorderAndCache()
47		{
48	3	$this->recorder = new Recorder($this->getDataFolder());
49
50	3	exec('rm -rf '.$this->getDataFolder());
51	3	exec('rm -rf '.$this->getCacheFolder());
52
53	3	if (!file_exists($this->getDataFolder())) {
54	3	mkdir($this->getDataFolder());
55	3	mkdir($this->getDataFolder().'/links');
56	3	mkdir($this->getCacheFolder());
57		}
58	3	}
59
60		public function setWaitBetweenRequest(int $microSeconds = 100)
61		{
62		$this->wai = $microSeconds;
		0 ignored issues – show Bug Best Practice introduced 2019-01-16 10:27 UTC by Report Bug Copy Issue Report The property `wai` does not exist. Although not strictly required by PHP, it is generally a best practice to declare properties explicitly. Loading history...
63		}
64
65	3	public function crawl(bool $debug = false)
66		{
67	3	$nothingUpdated = true;
68
69	3	if ($debug) {
70	3	echo PHP_EOL.PHP_EOL.'// -----'.PHP_EOL.'// '.$this->counter.' crawled / '
71	3	.count($this->urls).' found '.PHP_EOL.'// -----'.PHP_EOL;
72		}
73
74	3	foreach ($this->urls as $urlToParse => $url) {
75	3	if (null !== $url && (false === $url->can_be_crawled \|\| true === $url->can_be_crawled)) { // déjà crawlé
76		continue;
77		}
78
79	3	if ($debug) {
80	3	echo ' '.$urlToParse.PHP_EOL;
81		}
82
83	3	$nothingUpdated = false;
84	3	++$this->counter;
85
86	3	$harvest = $this->harvest($urlToParse);
87
88	3	$this->cacheRobotsTxt($harvest);
89	3	usleep($this->wait);
90		}
91
92	3	++$this->currentClick;
93
94	3	$record = $nothingUpdated \|\| $this->currentClick >= $this->limit;
95
96	3	return $record ? $this->recorder->record($this->urls) : $this->crawl($debug);
97		}
98
99	3	protected function cacheRobotsTxt($harvest)
100		{
101	3	if (null === $this->robotsTxt && $harvest instanceof Harvest) {
102	2	$this->robotsTxt = $harvest->getRobotsTxt();
103		}
104
105	3	return $this;
106		}
107
108	2	protected function loadRobotsTxt(Harvest $harvest)
109		{
110	2	if (null !== $this->robotsTxt) {
111		$harvest->setRobotsTxt($this->robotsTxt);
112		}
113
114	2	return $this;
115		}
116
117	2	protected function cache($harvest)
118		{
119	2	if (false === strpos($harvest->getResponse()->getContentType(), 'text/html')) {
120		return;
121		}
122
123	2	$url = ltrim($harvest->getAbsoluteInternalLink($harvest->getResponse()->getEffectiveUrl()), '/');
124	2	$urlPart = explode('/', $url);
125	2	$folder = $this->getCacheFolder();
126
127	2	$urlPartLenght = count($urlPart);
128	2	for ($i = 0; $i < $urlPartLenght; ++$i) {
129	2	if ($i == count($urlPart) - 1) {
130	2	$filename = empty($urlPart[$i]) ? 'index.html' : $urlPart[$i];
131	2	file_put_contents($folder.'/'.$filename, $harvest->getResponse()->getContent());
132		} else {
133		$folder .= '/'.$urlPart[$i];
134		if (!file_exists($folder)) {
135		mkdir($folder);
136		}
137		}
138		}
139	2	}
140
141	3	protected function harvest(string $urlToParse)
142		{
143	3	$url = $this->urls[$urlToParse] = $this->urls[$urlToParse] ?? new Url($urlToParse, $this->currentClick);
144
145	3	$url->updated_at = date('Ymd');
146	3	$url->can_be_crawled = $this->ignore->allows($urlToParse, $this->userAgent);
147
148	3	if (false === $url->can_be_crawled) {
149		return;
150		}
151
152	3	$harvest = Harvest::fromUrl($urlToParse, $this->userAgent);
153
154	3	if (!$harvest instanceof Harvest) {
155	1	$url->indexable = Indexable::NOT_INDEXABLE_NETWORK_ERROR;
156
157	1	return;
158		}
159
160	2	$this->loadRobotsTxt($harvest);
161
162	2	$url->indexable = $harvest->isIndexable();
163
164	2	if (Indexable::NOT_INDEXABLE_3XX === $url->indexable) {
165		$redir = $harvest->getRedirection();
166		if (false !== $redir) {
167		$links = Harvest::LINK_INTERNAL === $harvest->getType($redir) ? [$redir] : [];
		0 ignored issues – show introduced 2019-01-15 18:19 UTC by Report Bug Copy Issue Report The condition `PiedWeb\UrlHarvester\Har...arvest->getType($redir)` is always `false`. Loading history...
168		}
169		} else {
170	2	$this->cache($harvest);
171
172	2	$this->recorder->recordOutboundLink($url, $harvest->getLinks());
173
174	2	$url->links = count($harvest->getLinks());
175	2	$url->links_duplicate = $harvest->getNbrDuplicateLinks();
176	2	$url->links_internal = count($harvest->getLinks(Harvest::LINK_INTERNAL));
177	2	$url->links_self = count($harvest->getLinks(Harvest::LINK_SELF));
178	2	$url->links_sub = count($harvest->getLinks(Harvest::LINK_SUB));
179	2	$url->links_external = count($harvest->getLinks(Harvest::LINK_EXTERNAL));
180	2	$links = $harvest->getLinks(Harvest::LINK_INTERNAL);
181
182	2	$url->ratio_text_code = $harvest->getRatioTxtCode();
183	2	$url->load_time = $harvest->getResponse()->getInfo('total_time');
184	2	$url->size = $harvest->getResponse()->getInfo('size_download');
185
186	2	$breadcrumb = $harvest->getBreadCrumb();
187	2	if (is_array($breadcrumb)) {
188		$url->breadcrumb_level = count($breadcrumb);
189		$url->breadcrumb_first = isset($breadcrumb[1]) ? $breadcrumb[1]->getCleanName() : '';
190		$url->breadcrumb_text = $harvest->getBreadCrumb('//');
191		}
192
193	2	$url->title = $harvest->getUniqueTag('head title') ?? '';
194	2	$url->kws = ','.implode(',', $harvest->getKws()).',';
195	2	$url->h1 = $harvest->getUniqueTag('h1') ?? '';
196		}
197
198	2	if (isset($links)) {
199	2	foreach ($links as $link) {
200	2	$linkUrl = $link->getPageUrl();
201	2	$this->urls[$linkUrl] = $this->urls[$linkUrl] ?? new Url($linkUrl, ($this->currentClick + 1));
202	2	$this->recorder->recordInboundLink($url, $this->urls[$linkUrl]);
203	2	++$this->urls[$linkUrl]->inboundlinks;
204		}
205		}
206
207	2	return $harvest;
208		}
209		}
210

PiedWeb / SeoPocketCrawler

Push — master ( edfe10...b5c69a )

Crawler::loadRobotsTxt() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like