Scrapper::finalizeData() - Code Metrics - reliqarts/laravel-scavenger - Measure and Improve Code Quality continuously with Scrutinizer

Scrapper::finalizeData() A
last analyzed 2020-09-01 19:23 UTC

↳ Parent: Scrapper

Complexity

Conditions	1
Paths	1

Size

Total Lines	9
Code Lines	5

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
eloc	5
nc	1
nop	2
dl	0
loc	9
rs	10
c	0
b	0
f	0

<?php

declare(strict_types=1);

namespace ReliqArts\Scavenger\Service;

use Exception;
use Illuminate\Console\Command;
use Illuminate\Database\QueryException;
use Illuminate\Support\Collection;
use InvalidArgumentException;
use JsonException;
use Psr\Log\LoggerInterface;
use ReliqArts\Scavenger\Helper\FormattedMessage;
use ReliqArts\Scavenger\Helper\TargetKey;
use ReliqArts\Scavenger\Model\Scrap;
use ReliqArts\Scavenger\Model\Target;
use ReliqArts\Scavenger\TitleLink;
use Symfony\Component\DomCrawler\Crawler;

class Scrapper extends Communicator
{
    private const ERROR_INVALID_SPECIAL_KEY = '!#ERR - S.Key not valid!';
    private const KEY_PREFIX = ConfigProvider::SPECIAL_KEY_PREFIX;
    private const KEY_TARGET = self::KEY_PREFIX . 'target';
    private const KEY_ID = self::KEY_PREFIX . 'id';
    private const KEY_SERP_RESULT = self::KEY_PREFIX . 'serp_result';
    private const ENCODE_CHARSET = 'UTF-8';

    /**
     * @var array[]
     */
    private array $raw;
    private LoggerInterface $logger;
    private int $newScrapsCount;
    private Collection $relatedObjects;
    private Scanner $scanner;

    /**
     * Scraps found so far.
     */
    private Collection $scraps;

    /**
     * Scrapper constructor.
     */
    public function __construct(
        LoggerInterface $logger,
        ?Command $callingCommand = null,
        string $hashAlgorithm = self::HASH_ALGORITHM,
        int $verbosity = self::VERBOSITY_LOW
    ) {
        parent::__construct($callingCommand);

        $this->raw = [];
        $this->logger = $logger;
        $this->scanner = new Scanner();
        $this->hashAlgorithm = $hashAlgorithm;
        $this->scraps = collect([]);
        $this->relatedObjects = collect([]);
        $this->newScrapsCount = 0;
        $this->verbosity = $verbosity;
    }

    /**
     * Collect scrap array from crawler using markup.
     *
     * @throws JsonException
     */
    public function collect(TitleLink $titleLink, Target $target, Crawler $crawler): void
    {
        $markup = $target->getMarkup();
        $data = $this->initializeData(
            $titleLink,
            $target,
            $crawler,
            $markup[TargetKey::special(TargetKey::MARKUP_INSIDE)] ?? []
        );
        $data = $this->preprocess($data, $target);

        if (!$this->verifyData($data, $target)) {
            return;
        }

        $data = $this->finalizeData($data, $target);

        $this->raw[$data[self::KEY_ID]] = $data;
        $this->scraps->push($this->buildScrapFromData($data));

        // feedback
        $this->tell(
            FormattedMessage::get(FormattedMessage::SCRAP_GATHERED, $data[self::KEY_ID])
            . ($this->verbosity >= self::VERBOSITY_HIGH ? '-- ' . json_encode($data, JSON_THROW_ON_ERROR, 512) : null)
        );
    }

    public function convertScraps(bool $convertDuplicates = false, bool $storeRelatedReferences = false): void
    {
        $this->scraps->map(
            function (Scrap $scrap) use ($convertDuplicates, $storeRelatedReferences) {
                try {
                    $relatedObject = $scrap->convert($convertDuplicates, $storeRelatedReferences);
                    if ($relatedObject !== null) {
                        $this->relatedObjects->push($relatedObject);
                    }
                } catch (QueryException $e) {
                    $this->logger->warning($e, ['scrap' => $scrap]);
                }
            }
        );
    }

    public function getNewScrapsCount(): int
    {
        return $this->newScrapsCount;
    }

    public function getScraps(): Collection
    {
        return $this->scraps;
    }

    public function saveScraps(): void
    {
        $this->scraps->map(
            function (Scrap $scrap): void {
                try {
                    $scrap->save();
                } catch (QueryException $e) {
                    $errorMessage = FormattedMessage::get(
                        FormattedMessage::SCRAP_SAVE_EXCEPTION,
                        $scrap->hash,
                        $e->getMessage()
                    );
                    if ($this->verbosity >= self::VERBOSITY_HIGH) {
                        $errorMessage .= ' -- ' . $scrap->toJson();
                    }

                    $this->tell($errorMessage);
                    $this->logger->error($errorMessage);
                }
            }
        );
    }

    /**
     * Retrieve related objects which are a result of scrap conversion.
     */
    public function getRelatedObjects(): Collection
    {
        return $this->relatedObjects;
    }

    /**
     * Initialize data.
     *
     * @noinspection PhpTooManyParametersInspection
     */
    private function initializeData(
        TitleLink $titleLink,
        Target $target,
        Crawler $crawler,
        array $markupOverride = []
    ): array {
        $markup = !empty($markupOverride) ? $markupOverride : $target->getMarkup();

        $data[TargetKey::TITLE] = $titleLink->getTitle();

        $data[TargetKey::special(TargetKey::LINK)] = $titleLink->getLink();
        $data[TargetKey::special(TargetKey::POSITION)] = $target->getCursor();
        $data[TargetKey::special(TargetKey::SOURCE)] = $titleLink->getLink();
        $data[TargetKey::special(TargetKey::MODEL)] = $target->getModel();

        // build initial scrap data from markup and dissect
        foreach ($markup as $attr => $path) {
            if (ConfigProvider::isSpecialKey($path)) {
                // path is special key, use special key value from scrap
                $data[$attr] = !empty($data[$path]) ? $data[$path] : self::ERROR_INVALID_SPECIAL_KEY;
            } elseif (!ConfigProvider::isSpecialKey($attr)) {
                try {
                    $attrCrawler = $crawler->filter($path);
                    $data[$attr] = $attr === TargetKey::TITLE ? $attrCrawler->text() : $attrCrawler->html();

                    // split single attributes into multiple based on regex
                    if (!empty($target->getDissect()[$attr])) {
                        $dissectMap = $target->getDissect()[$attr];

                        // check _retain meta property
                        // to determine whether details should be left in source attribute after extraction
                        $retain = empty($dissectMap[TargetKey::special(TargetKey::RETAIN)])
                            ? false
                            : $dissectMap[TargetKey::special(TargetKey::RETAIN)];
                        unset($dissectMap[TargetKey::special(TargetKey::RETAIN)]);

                        // Extract details into scrap
                        $data = array_merge($data, Scanner::pluckDetails($data[$attr], $dissectMap, $retain));

                        // unset dissectMap
                        unset($dissectMap);
                    }
                } catch (InvalidArgumentException $e) {
                    $exMessage = FormattedMessage::get(
                        FormattedMessage::EXCEPTION_THROWN_FOR_ATTRIBUTE,
                        $attr,
                        $target->getName(),
                        $e->getMessage()
                    );
                    $this->tell($exMessage);
                    $this->logger->warning($exMessage, [$markup[$attr]]);
                }
            }
            unset($path);
        }

        return $data;
    }

    private function preprocess(array $data, Target $target): array
    {
        // preprocess and remap scrap data parts
        foreach ($data as $attr => $value) {
            $data[$attr] = $this->encodeAttribute($data, $attr);

            // preprocess
            if (!empty($target->getPreprocess()[$attr])) {
                $preprocess = $target->getPreprocess()[$attr];
                // check for optional third parameter of array, which indicates that callable method needs an instance
                if (is_array($preprocess) && isset($preprocess[2])) {
                    // if callable needs instance, resolve object
                    if ($preprocess[2]) {
                        $preprocess[0] = resolve($preprocess[0]);
                    }
                    unset($preprocess[2]);
                }
                // if preprocess is callable call it on attribute value
                if (is_callable($preprocess)) {
                    try {
                        $data[$attr] = $preprocess($data[$attr]);
                    } catch (Exception $e) {
                        $this->logger->error($e);
                    }
                } else {
                    $this->logger->warning(
                        FormattedMessage::get(
                            FormattedMessage::PREPROCESS_NOT_CALLABLE,
                            $attr,
                            $target->getName()
                        )
                    );
                }
            }

            // remap entity attribute name if specified
            if (!empty($target->getRemap()[$attr])) {
                $newAttrName = $target->getRemap()[$attr];
                $data[$newAttrName] = !empty($data[$attr]) ? $data[$attr] : null;

                if ($attr !== TargetKey::TITLE) {
                    unset($data[$attr]);
                }
            }
        }

        return $data;
    }

    private function encodeAttribute(array $data, string $attr): string
    {
        $attributeText = (string)$data[$attr];

        // ensure title has UC words
        if ($attr === TargetKey::TITLE) {
            return utf8_encode(ucwords(mb_strtolower($attributeText)));
        }

        return iconv(
            mb_detect_encoding($attributeText, mb_detect_order(), true),
            self::ENCODE_CHARSET,
            $attributeText
        );
    }

    /**
     * @throws JsonException
     */
    private function verifyData(array $data, Target $target): bool
    {
        if ($this->scanner->hasBadWords($data, $target->getBadWords())) {
            $badWordMessage = sprintf(
                FormattedMessage::SCRAP_CONTAINS_BAD_WORD,
                json_encode($data, JSON_THROW_ON_ERROR, 512)
            );
            $this->logger->notice($badWordMessage);
            if ($this->verbosity >= self::VERBOSITY_HIGH) {
                $this->tell($badWordMessage);
            }

            return false;
        }

        return true;
    }

    /**
     * Finalize scrap.
     */
    private function finalizeData(array $data, Target $target): array
    {
        $data[self::KEY_ID] = hash($this->hashAlgorithm, json_encode($data, JSON_THROW_ON_ERROR, 512));
        $data[self::KEY_SERP_RESULT] = $target->isSearchEngineRequestPages();
        $data[self::KEY_TARGET] = $target->getName();

        ksort($data);

        return $data;
    }

    private function buildScrapFromData(array $data): Scrap
    {
        $scrap = Scrap::firstOrNew(
            [
                'hash' => $data[self::KEY_ID],
            ],
            [
                'model' => $data[TargetKey::special(TargetKey::MODEL)],
                'source' => $data[TargetKey::special(TargetKey::SOURCE)],
                'title' => $data[TargetKey::TITLE],
                'data' => json_encode($data),
            ]
        );

        if (!$scrap->exists) {
            ++$this->newScrapsCount;
        }

        return $scrap;
    }
}


1			<?php
2
3			declare(strict_types=1);
4
5			namespace ReliqArts\Scavenger\Service;
6
7			use Exception;
8			use Illuminate\Console\Command;
9			use Illuminate\Database\QueryException;
10			use Illuminate\Support\Collection;
11			use InvalidArgumentException;
12			use JsonException;
13			use Psr\Log\LoggerInterface;
14			use ReliqArts\Scavenger\Helper\FormattedMessage;
15			use ReliqArts\Scavenger\Helper\TargetKey;
16			use ReliqArts\Scavenger\Model\Scrap;
17			use ReliqArts\Scavenger\Model\Target;
18			use ReliqArts\Scavenger\TitleLink;
19			use Symfony\Component\DomCrawler\Crawler;
20
21			class Scrapper extends Communicator
22			{
23			private const ERROR_INVALID_SPECIAL_KEY = '!#ERR - S.Key not valid!';
24			private const KEY_PREFIX = ConfigProvider::SPECIAL_KEY_PREFIX;
25			private const KEY_TARGET = self::KEY_PREFIX . 'target';
26			private const KEY_ID = self::KEY_PREFIX . 'id';
27			private const KEY_SERP_RESULT = self::KEY_PREFIX . 'serp_result';
28			private const ENCODE_CHARSET = 'UTF-8';
29
30			/**
31			* @var array[]
32			*/
33			private array $raw;
34			private LoggerInterface $logger;
35			private int $newScrapsCount;
36			private Collection $relatedObjects;
37			private Scanner $scanner;
38
39			/**
40			* Scraps found so far.
41			*/
42			private Collection $scraps;
43
44			/**
45			* Scrapper constructor.
46			*/
47			public function __construct(
48			LoggerInterface $logger,
49			?Command $callingCommand = null,
50			string $hashAlgorithm = self::HASH_ALGORITHM,
51			int $verbosity = self::VERBOSITY_LOW
52			) {
53			parent::__construct($callingCommand);
54
55			$this->raw = [];
56			$this->logger = $logger;
57			$this->scanner = new Scanner();
58			$this->hashAlgorithm = $hashAlgorithm;
59			$this->scraps = collect([]);
60			$this->relatedObjects = collect([]);
61			$this->newScrapsCount = 0;
62			$this->verbosity = $verbosity;
63			}
64
65			/**
66			* Collect scrap array from crawler using markup.
67			*
68			* @throws JsonException
69			*/
70			public function collect(TitleLink $titleLink, Target $target, Crawler $crawler): void
71			{
72			$markup = $target->getMarkup();
73			$data = $this->initializeData(
74			$titleLink,
75			$target,
76			$crawler,
77			$markup[TargetKey::special(TargetKey::MARKUP_INSIDE)] ?? []
78			);
79			$data = $this->preprocess($data, $target);
80
81			if (!$this->verifyData($data, $target)) {
82			return;
83			}
84
85			$data = $this->finalizeData($data, $target);
86
87			$this->raw[$data[self::KEY_ID]] = $data;
88			$this->scraps->push($this->buildScrapFromData($data));
89
90			// feedback
91			$this->tell(
92			FormattedMessage::get(FormattedMessage::SCRAP_GATHERED, $data[self::KEY_ID])
93			. ($this->verbosity >= self::VERBOSITY_HIGH ? '-- ' . json_encode($data, JSON_THROW_ON_ERROR, 512) : null)
94			);
95			}
96
97			public function convertScraps(bool $convertDuplicates = false, bool $storeRelatedReferences = false): void
98			{
99			$this->scraps->map(
100			function (Scrap $scrap) use ($convertDuplicates, $storeRelatedReferences) {
101			try {
102			$relatedObject = $scrap->convert($convertDuplicates, $storeRelatedReferences);
103			if ($relatedObject !== null) {
104			$this->relatedObjects->push($relatedObject);
105			}
106			} catch (QueryException $e) {
107			$this->logger->warning($e, ['scrap' => $scrap]);
108			}
109			}
110			);
111			}
112
113			public function getNewScrapsCount(): int
114			{
115			return $this->newScrapsCount;
116			}
117
118			public function getScraps(): Collection
119			{
120			return $this->scraps;
121			}
122
123			public function saveScraps(): void
124			{
125			$this->scraps->map(
126			function (Scrap $scrap): void {
127			try {
128			$scrap->save();
129			} catch (QueryException $e) {
130			$errorMessage = FormattedMessage::get(
131			FormattedMessage::SCRAP_SAVE_EXCEPTION,
132			$scrap->hash,
133			$e->getMessage()
134			);
135			if ($this->verbosity >= self::VERBOSITY_HIGH) {
136			$errorMessage .= ' -- ' . $scrap->toJson();
137			}
138
139			$this->tell($errorMessage);
140			$this->logger->error($errorMessage);
141			}
142			}
143			);
144			}
145
146			/**
147			* Retrieve related objects which are a result of scrap conversion.
148			*/
149			public function getRelatedObjects(): Collection
150			{
151			return $this->relatedObjects;
152			}
153
154			/**
155			* Initialize data.
156			*
157			* @noinspection PhpTooManyParametersInspection
158			*/
159			private function initializeData(
160			TitleLink $titleLink,
161			Target $target,
162			Crawler $crawler,
163			array $markupOverride = []
164			): array {
165			$markup = !empty($markupOverride) ? $markupOverride : $target->getMarkup();
166
167			$data[TargetKey::TITLE] = $titleLink->getTitle();
			0 ignored issues – show Comprehensibility Best Practice introduced 2020-02-26 20:16 UTC by Report Bug Copy Issue Report `$data` was never initialized. Although not strictly required by PHP, it is generally a good practice to add `$data = array();` before regardless. Loading history...
168			$data[TargetKey::special(TargetKey::LINK)] = $titleLink->getLink();
169			$data[TargetKey::special(TargetKey::POSITION)] = $target->getCursor();
170			$data[TargetKey::special(TargetKey::SOURCE)] = $titleLink->getLink();
171			$data[TargetKey::special(TargetKey::MODEL)] = $target->getModel();
172
173			// build initial scrap data from markup and dissect
174			foreach ($markup as $attr => $path) {
175			if (ConfigProvider::isSpecialKey($path)) {
176			// path is special key, use special key value from scrap
177			$data[$attr] = !empty($data[$path]) ? $data[$path] : self::ERROR_INVALID_SPECIAL_KEY;
178			} elseif (!ConfigProvider::isSpecialKey($attr)) {
179			try {
180			$attrCrawler = $crawler->filter($path);
181			$data[$attr] = $attr === TargetKey::TITLE ? $attrCrawler->text() : $attrCrawler->html();
182
183			// split single attributes into multiple based on regex
184			if (!empty($target->getDissect()[$attr])) {
185			$dissectMap = $target->getDissect()[$attr];
186
187			// check _retain meta property
188			// to determine whether details should be left in source attribute after extraction
189			$retain = empty($dissectMap[TargetKey::special(TargetKey::RETAIN)])
190			? false
191			: $dissectMap[TargetKey::special(TargetKey::RETAIN)];
192			unset($dissectMap[TargetKey::special(TargetKey::RETAIN)]);
193
194			// Extract details into scrap
195			$data = array_merge($data, Scanner::pluckDetails($data[$attr], $dissectMap, $retain));
196
197			// unset dissectMap
198			unset($dissectMap);
199			}
200			} catch (InvalidArgumentException $e) {
201			$exMessage = FormattedMessage::get(
202			FormattedMessage::EXCEPTION_THROWN_FOR_ATTRIBUTE,
203			$attr,
204			$target->getName(),
205			$e->getMessage()
206			);
207			$this->tell($exMessage);
208			$this->logger->warning($exMessage, [$markup[$attr]]);
209			}
210			}
211			unset($path);
212			}
213
214			return $data;
215			}
216
217			private function preprocess(array $data, Target $target): array
218			{
219			// preprocess and remap scrap data parts
220			foreach ($data as $attr => $value) {
221			$data[$attr] = $this->encodeAttribute($data, $attr);
222
223			// preprocess
224			if (!empty($target->getPreprocess()[$attr])) {
225			$preprocess = $target->getPreprocess()[$attr];
226			// check for optional third parameter of array, which indicates that callable method needs an instance
227			if (is_array($preprocess) && isset($preprocess[2])) {
228			// if callable needs instance, resolve object
229			if ($preprocess[2]) {
230			$preprocess[0] = resolve($preprocess[0]);
231			}
232			unset($preprocess[2]);
233			}
234			// if preprocess is callable call it on attribute value
235			if (is_callable($preprocess)) {
236			try {
237			$data[$attr] = $preprocess($data[$attr]);
238			} catch (Exception $e) {
239			$this->logger->error($e);
240			}
241			} else {
242			$this->logger->warning(
243			FormattedMessage::get(
244			FormattedMessage::PREPROCESS_NOT_CALLABLE,
245			$attr,
246			$target->getName()
247			)
248			);
249			}
250			}
251
252			// remap entity attribute name if specified
253			if (!empty($target->getRemap()[$attr])) {
254			$newAttrName = $target->getRemap()[$attr];
255			$data[$newAttrName] = !empty($data[$attr]) ? $data[$attr] : null;
256
257			if ($attr !== TargetKey::TITLE) {
258			unset($data[$attr]);
259			}
260			}
261			}
262
263			return $data;
264			}
265
266			private function encodeAttribute(array $data, string $attr): string
267			{
268			$attributeText = (string)$data[$attr];
269
270			// ensure title has UC words
271			if ($attr === TargetKey::TITLE) {
272			return utf8_encode(ucwords(mb_strtolower($attributeText)));
273			}
274
275			return iconv(
276			mb_detect_encoding($attributeText, mb_detect_order(), true),
277			self::ENCODE_CHARSET,
278			$attributeText
279			);
280			}
281
282			/**
283			* @throws JsonException
284			*/
285			private function verifyData(array $data, Target $target): bool
286			{
287			if ($this->scanner->hasBadWords($data, $target->getBadWords())) {
288			$badWordMessage = sprintf(
289			FormattedMessage::SCRAP_CONTAINS_BAD_WORD,
290			json_encode($data, JSON_THROW_ON_ERROR, 512)
291			);
292			$this->logger->notice($badWordMessage);
293			if ($this->verbosity >= self::VERBOSITY_HIGH) {
294			$this->tell($badWordMessage);
295			}
296
297			return false;
298			}
299
300			return true;
301			}
302
303			/**
304			* Finalize scrap.
305			*/
306			private function finalizeData(array $data, Target $target): array
307			{
308			$data[self::KEY_ID] = hash($this->hashAlgorithm, json_encode($data, JSON_THROW_ON_ERROR, 512));
309			$data[self::KEY_SERP_RESULT] = $target->isSearchEngineRequestPages();
310			$data[self::KEY_TARGET] = $target->getName();
311
312			ksort($data);
313
314			return $data;
315			}
316
317			private function buildScrapFromData(array $data): Scrap
318			{
319			$scrap = Scrap::firstOrNew(
320			[
321			'hash' => $data[self::KEY_ID],
322			],
323			[
324			'model' => $data[TargetKey::special(TargetKey::MODEL)],
325			'source' => $data[TargetKey::special(TargetKey::SOURCE)],
326			'title' => $data[TargetKey::TITLE],
327			'data' => json_encode($data),
328			]
329			);
330
331			if (!$scrap->exists) {
332			++$this->newScrapsCount;
333			}
334
335			return $scrap;
336			}
337			}
338

reliqarts / laravel-scavenger

Scrapper::finalizeData() A last analyzed 2020-09-01 19:23 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

Scrapper::finalizeData() A
last analyzed 2020-09-01 19:23 UTC