Crawler::crawl() - Code Metrics - pilipinews/gma - Measure and Improve Code Quality continuously with Scrutinizer

Crawler::crawl() A
last analyzed 2020-07-07 01:33 UTC

↳ Parent: Crawler

Complexity

Conditions	1
Paths	1

Size

Total Lines	9
Code Lines	4

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	5
CRAP Score	1

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	1
eloc	4
c	1
b	0
f	0
nc	1
nop	0
dl	0
loc	9
ccs	5
cts	5
cp	1
crap	1
rs	10

<?php

namespace Pilipinews\Website\Gma;

use Pilipinews\Common\Client;
use Pilipinews\Common\Crawler as DomCrawler;
use Pilipinews\Common\Interfaces\CrawlerInterface;

/**
 * GMA News Crawler
 *
 * @package Pilipinews
 * @author  Rougin Gutib <[email protected]>
 */
class Crawler implements CrawlerInterface
{
    /**
     * @var string[]
     */
    protected $allowed = array('news/nation', 'news/regions');

    /**
     * @var string
     */
    protected $link = 'https://data.gmanetwork.com/gno/pages/home_1a_json.gz';

    /**
     * Returns an array of articles to scrape.
     *
     * @return string[]
     */
    public function crawl()
    {
        $response = Client::request($this->link);

        $json = json_decode($response, true);

        $items = $json['story_lists_just_in'];

        return $this->verify((array) $items);
    }

    /**
     * Returns the allowed article URLs to scrape.
     *
     * @param  string[] $items
     * @return string[]
     */
    protected function verify($items)
    {
        $base = 'https://www.gmanetwork.com/news/';

        $allowed = (array) $this->allowed;

        $callback = function ($item) use ($base, $allowed)
        {
            $link = null;

            foreach ((array) $allowed as $keyword)
            {
                $result = strpos($item['link'], (string) $keyword);

                $result !== false && $link = $base . $item['link'];
            }

            return (string) $link;
        };

        $result = array_map($callback, (array) $items);

        return array_values(array_filter($result));
    }
}


1		<?php
2
3		namespace Pilipinews\Website\Gma;
4
5		use Pilipinews\Common\Client;
6		use Pilipinews\Common\Crawler as DomCrawler;
7		use Pilipinews\Common\Interfaces\CrawlerInterface;
8
9		/**
10		* GMA News Crawler
11		*
12		* @package Pilipinews
13		* @author Rougin Gutib <[email protected]>
14		*/
15		class Crawler implements CrawlerInterface
16		{
17		/**
18		* @var string[]
19		*/
20		protected $allowed = array('news/nation', 'news/regions');
21
22		/**
23		* @var string
24		*/
25		protected $link = 'https://data.gmanetwork.com/gno/pages/home_1a_json.gz';
26
27		/**
28		* Returns an array of articles to scrape.
29		*
30		* @return string[]
31		*/
32	3	public function crawl()
33		{
34	3	$response = Client::request($this->link);
35
36	3	$json = json_decode($response, true);
37
38	3	$items = $json['story_lists_just_in'];
39
40	3	return $this->verify((array) $items);
41		}
42
43		/**
44		* Returns the allowed article URLs to scrape.
45		*
46		* @param string[] $items
47		* @return string[]
48		*/
49	3	protected function verify($items)
50		{
51	3	$base = 'https://www.gmanetwork.com/news/';
52
53	3	$allowed = (array) $this->allowed;
54
55	3	$callback = function ($item) use ($base, $allowed)
56		{
57	3	$link = null;
58
59	3	foreach ((array) $allowed as $keyword)
60		{
61	3	$result = strpos($item['link'], (string) $keyword);
62
63	3	$result !== false && $link = $base . $item['link'];
64	1	}
65
66	3	return (string) $link;
67	3	};
68
69	1	$result = array_map($callback, (array) $items);
70
71	1	return array_values(array_filter($result));
72		}
73		}
74

pilipinews / gma

Crawler::crawl() A last analyzed 2020-07-07 01:33 UTC

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like

Crawler::crawl() A
last analyzed 2020-07-07 01:33 UTC