Crawler::crawl() - Code Metrics - bilginpro/agency-dha - Measure and Improve Code Quality continuously with Scrutinizer

Crawler::crawl() B
last analyzed 2019-05-25 13:39 UTC

↳ Parent: Crawler

Complexity

Conditions	7
Paths	6

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	0
CRAP Score	56

Importance

Changes

Metric	Value
dl	0
loc	33
ccs	0
cts	29
cp	0
rs	8.4586
c	0
b	0
f	0
cc	7
nc	6
nop	1
crap	56

<?php

namespace BilginPro\Agency\Dha;

use Carbon\Carbon;
use GuzzleHttp;

/**
 * Class Crawler
 * @package BilginPro\Ajans\Dha
 */
class Crawler
{
    /**
     * @var string
     */
    protected $x_code = '';

    /**
     * @var string
     */
    protected $y_code = '';

    /**
     * @var int
     */
    protected $summary_length = 150;

    /**
     * @var array
     */
    protected $attributes = [
        'limit' => '5',
    ];

    /**
     * Create a new Crawler Instance
     */
    public function __construct($config)
    {
        $this->setParameters($config);
    }

    /**
     * Does the magic.
     * @return array
     */
    public function crawl($attributes = [])
    {
        $this->setAttributes($attributes);

        $response = $this->fetchUrl($this->getUrl());
        $xml = new \SimpleXMLElement($response);
        $result = [];
        $i = 0;
        foreach ($xml->channel->item as $item) {
            if ($this->attributes['limit'] > $i) {
                $news = new \stdClass;
                $news->code = (string)$item->guid;
                $news->title = (string)$item->title;
                $news->summary = $this->createSummary($item->description);
                $news->content = (string)trim(preg_replace('/\s+/', ' ', $item->description));
                // preg_replace is for cleaning newlines.
                $news->created_at = (new Carbon($item->pubDate))->format('d.m.Y H:i:s');
                $news->category = $this->titleCase(str_replace('DHA-', '', $item->category));
                $news->city = (!empty($item->location) ? $this->titleCase($item->location) : '');
                $news->images = [];
                if (isset($item->photoshd) && count($item->photoshd) > 0) {
                    foreach ($item->photoshd as $image) {
                        $news->images[] = (string)$image;
                    }
                }

                $result[] = $news;
                $i++;
            }
        }

        return $result;
    }

    /**
     * Creates short summary of the news, strip credits.
     * @param $text
     * @return string
     */
    public function createSummary($text)
    {
        if (strpos($text, '(DHA)') > 0) {
            $split = explode('(DHA)', $text);
            if (count($split) > 1) {
                $text = $split[1];
                $text = trim($text, ' \t\n\r\0\x0B-');
            }
        }
        $summary = (string)$this->shortenString(strip_tags($text), $this->summary_length);

        return $summary;
    }

    /**
     * Sets config parameters.
     */
    public function setParameters($config)
    {
        if (!is_array($config)) {
            throw new \InvalidArgumentException('$config variable must be an array.');
        }
        if (array_key_exists('x_code', $config)) {
            $this->x_code = $config['x_code'];
        }
        if (array_key_exists('y_code', $config)) {
            $this->y_code = $config['y_code'];
        }
        if (array_key_exists('limit', $config)) {
            $this->limit = $config['limit'];
class MyClass { }

$x = new MyClass();
$x->foo = true;
        }
    }

    /**
     * Sets filter attributes.
     * @param $attributes array
     */
    protected function setAttributes($attributes)
    {
        foreach ($attributes as $key => $value) {
            $this->attributes[$key] = $value;
        }
    }

    /**
     * Returns full url for crawling.
     * @return string
     */
    public function getUrl()
    {
        $url = 'http://ajans.dha.com.tr/dhayharss_resimli.php'
            . '?x=' . $this->x_code
            . '&y=' . $this->y_code;

        return $url;
    }


    /**
     * Fethches given url and returns response as string.
     * @param $url
     * @param string $method
     * @param array $options
     *
     * @return string
     */
    public function fetchUrl($url, $method = 'GET', $options = [])
    {
        $client = new GuzzleHttp\Client();
        $res = $client->request($method, $url, $options);
        if ($res->getStatusCode() == 200) {
            return (string)$res->getBody();
        }
        return '';
    }

    /**
     * Cuts the given string from the end of the appropriate word.
     * @param $str
     * @param $len
     * @return string
     */
    public function shortenString($str, $len)
    {
        if (strlen($str) > $len) {
            $str = rtrim(mb_substr($str, 0, $len, 'UTF-8'));
            $str = substr($str, 0, strrpos($str, ' '));
            $str .= '...';
            $str = str_replace(',...', '...', $str);
        }
        return $str;
    }

    /**
     * Converts a string to "Title Case"
     * @param $str
     * @return string
     */
    public function titleCase($str)
    {
        $str = mb_convert_case($str, MB_CASE_TITLE, 'UTF-8');
        return $str;
    }
}


1			<?php
2
3			namespace BilginPro\Agency\Dha;
4
5			use Carbon\Carbon;
6			use GuzzleHttp;
7
8			/**
9			* Class Crawler
10			* @package BilginPro\Ajans\Dha
11			*/
12			class Crawler
13			{
14			/**
15			* @var string
16			*/
17			protected $x_code = '';
18
19			/**
20			* @var string
21			*/
22			protected $y_code = '';
23
24			/**
25			* @var int
26			*/
27			protected $summary_length = 150;
28
29			/**
30			* @var array
31			*/
32			protected $attributes = [
33			'limit' => '5',
34			];
35
36			/**
37			* Create a new Crawler Instance
38			*/
39			public function __construct($config)
40			{
41			$this->setParameters($config);
42			}
43
44			/**
45			* Does the magic.
46			* @return array
47			*/
48			public function crawl($attributes = [])
49			{
50			$this->setAttributes($attributes);
51
52			$response = $this->fetchUrl($this->getUrl());
53			$xml = new \SimpleXMLElement($response);
54			$result = [];
55			$i = 0;
56			foreach ($xml->channel->item as $item) {
57			if ($this->attributes['limit'] > $i) {
58			$news = new \stdClass;
59			$news->code = (string)$item->guid;
60			$news->title = (string)$item->title;
61			$news->summary = $this->createSummary($item->description);
62			$news->content = (string)trim(preg_replace('/\s+/', ' ', $item->description));
63			// preg_replace is for cleaning newlines.
64			$news->created_at = (new Carbon($item->pubDate))->format('d.m.Y H:i:s');
65			$news->category = $this->titleCase(str_replace('DHA-', '', $item->category));
66			$news->city = (!empty($item->location) ? $this->titleCase($item->location) : '');
67			$news->images = [];
68			if (isset($item->photoshd) && count($item->photoshd) > 0) {
69			foreach ($item->photoshd as $image) {
70			$news->images[] = (string)$image;
71			}
72			}
73
74			$result[] = $news;
75			$i++;
76			}
77			}
78
79			return $result;
80			}
81
82			/**
83			* Creates short summary of the news, strip credits.
84			* @param $text
85			* @return string
86			*/
87			public function createSummary($text)
88			{
89			if (strpos($text, '(DHA)') > 0) {
90			$split = explode('(DHA)', $text);
91			if (count($split) > 1) {
92			$text = $split[1];
93			$text = trim($text, ' \t\n\r\0\x0B-');
94			}
95			}
96			$summary = (string)$this->shortenString(strip_tags($text), $this->summary_length);
97
98			return $summary;
99			}
100
101			/**
102			* Sets config parameters.
103			*/
104			public function setParameters($config)
105			{
106			if (!is_array($config)) {
107			throw new \InvalidArgumentException('$config variable must be an array.');
108			}
109			if (array_key_exists('x_code', $config)) {
110			$this->x_code = $config['x_code'];
111			}
112			if (array_key_exists('y_code', $config)) {
113			$this->y_code = $config['y_code'];
114			}
115			if (array_key_exists('limit', $config)) {
116			$this->limit = $config['limit'];
			0 ignored issues – show Bug introduced 2017-09-02 14:25 UTC by Report Bug Copy Issue Report The property `limit` does not exist. Did you maybe forget to declare it? In PHP it is possible to write to properties without declaring them. For example, the following is perfectly valid PHP code: class MyClass { } $x = new MyClass(); $x->foo = true; Generally, it is a good practice to explictly declare properties to avoid accidental typos and provide IDE auto-completion: class MyClass { public $foo; } $x = new MyClass(); $x->foo = true; Loading history...
117			}
118			}
119
120			/**
121			* Sets filter attributes.
122			* @param $attributes array
123			*/
124			protected function setAttributes($attributes)
125			{
126			foreach ($attributes as $key => $value) {
127			$this->attributes[$key] = $value;
128			}
129			}
130
131			/**
132			* Returns full url for crawling.
133			* @return string
134			*/
135			public function getUrl()
136			{
137			$url = 'http://ajans.dha.com.tr/dhayharss_resimli.php'
138			. '?x=' . $this->x_code
139			. '&y=' . $this->y_code;
140
141			return $url;
142			}
143
144
145			/**
146			* Fethches given url and returns response as string.
147			* @param $url
148			* @param string $method
149			* @param array $options
150			*
151			* @return string
152			*/
153			public function fetchUrl($url, $method = 'GET', $options = [])
154			{
155			$client = new GuzzleHttp\Client();
156			$res = $client->request($method, $url, $options);
157			if ($res->getStatusCode() == 200) {
158			return (string)$res->getBody();
159			}
160			return '';
161			}
162
163			/**
164			* Cuts the given string from the end of the appropriate word.
165			* @param $str
166			* @param $len
167			* @return string
168			*/
169			public function shortenString($str, $len)
170			{
171			if (strlen($str) > $len) {
172			$str = rtrim(mb_substr($str, 0, $len, 'UTF-8'));
173			$str = substr($str, 0, strrpos($str, ' '));
174			$str .= '...';
175			$str = str_replace(',...', '...', $str);
176			}
177			return $str;
178			}
179
180			/**
181			* Converts a string to "Title Case"
182			* @param $str
183			* @return string
184			*/
185			public function titleCase($str)
186			{
187			$str = mb_convert_case($str, MB_CASE_TITLE, 'UTF-8');
188			return $str;
189			}
190			}
191

bilginpro / agency-dha

GitHub Access Token became invalid

Crawler::crawl() B last analyzed 2019-05-25 13:39 UTC

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like

Crawler::crawl() B
last analyzed 2019-05-25 13:39 UTC