Goose\Crawler

Complexity

Total Complexity

7

Size/Duplication

Total Lines	90
Duplicated Lines	0 %

Importance

Changes	9
Bugs	1	Features	0

5 Methods

Rating	Name	Duplication	Size	Complexity
A	__construct()	0	2	1
A	config()	0	2	1
A	modules()	0	9	2
A	crawl()	0	37	2
A	getDocument()	0	5	1

<?php declare(strict_types=1);

namespace Goose;

use Goose\Utils\Helper;
use DOMWrap\Document;

/**
 * Crawler
 *
 * @package Goose
 * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License 2.0
 */
class Crawler {
    /** @var Configuration */
    protected $config;

    /**
     * @param Configuration $config
     */
    public function __construct(Configuration $config) {
        $this->config = $config;
    }

    /**
     * @return Configuration
     */
    public function config(): Configuration {
        return $this->config;
    }

    /**
     * @param string $url
     * @param string|null $rawHTML
     *
     * @return Article
     */
    public function crawl(string $url, string $rawHTML = null): ?Article {
        $article = new Article();

        $parseCandidate = Helper::getCleanedUrl($url);

        $xmlInternalErrors = libxml_use_internal_errors(true);

        if (empty($rawHTML)) {
            $guzzle = new \GuzzleHttp\Client();
            $response = $guzzle->get($parseCandidate->url, $this->config()->get('browser'));
            $article->setRawResponse($response);
            $rawHTML = $response->getBody()->getContents();
        }

        // Generate document
        $doc = $this->getDocument($rawHTML);

        // Set core mutators
        $article->setFinalUrl($parseCandidate->url);
        $article->setDomain($parseCandidate->parts->host);
        $article->setLinkhash($parseCandidate->linkhash);
        $article->setRawHtml($rawHTML);
        $article->setDoc($doc);
        $article->setRawDoc(clone $doc);

        // Pre-extraction document cleaning
        $this->modules('cleaners', $article);

        // Extract content
        $this->modules('extractors', $article);

        // Post-extraction content formatting
        $this->modules('formatters', $article);

        libxml_use_internal_errors($xmlInternalErrors);

        return $article;
    }

    /**
     * @param string $rawHTML
     *
     * @return Document
     */
    private function getDocument(string $rawHTML): Document {
        $doc = new Document();
        $doc->html($rawHTML);

        return $doc;
    }

    /**
     * @param string $category
     * @param Article $article
     *
     * @return self
     */
    public function modules(string $category, Article $article): self {
        $modules = $this->config->getModules($category);

        foreach ($modules as $module) {
            $obj = new $module($this->config());
            $obj->run($article);
        }

        return $this;
    }
}