CrawlerDetect::getRegex() - Code Metrics - Inspection of "Increase performance by over 75%" - JayBizzle/Crawler-Detect - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#42)

by Mark

created 2015-12-10 16:30 UTC

CrawlerDetect::getRegex() A

↳ Parent: CrawlerDetect

Complexity

Conditions	1
Paths	1

Size

Total Lines	4
Code Lines	2

Duplication

Lines	0
Ratio	0 %

Importance

Changes	3
Bugs	1	Features	0

Metric	Value
c	3
b	1
f	0
dl	0
loc	4
rs	10
cc	1
eloc	2
nc	1
nop	0

<?php

namespace Jaybizzle\CrawlerDetect;

class CrawlerDetect
{
    protected $userAgent = null;

    protected $httpHeaders = array();

    protected $matches = array();

    /**
     * List of strings to remove from the user agent before running the crawler regex
     * Over a large list of user agents, this gives us about a 55% speed increase!
     * 
     * @var array
     */
    protected static $ignore = array(
        'Safari.[\d\.]*',
        'Firefox.[\d\.]*',
        'Chrome.[\d\.]*',
        'Chromium.[\d\.]*',
        'MSIE.[\d\.]',
        'Opera\/[\d\.]*',
        'Mozilla.[\d\.]*',
        'AppleWebKit.[\d\.]*',
        'Trident.[\d\.]*',
        'Windows NT.[\d\.]*',
        'Android.[\d\.]*',
        'Macintosh.',
        'Ubuntu',
        'Linux',
        'Intel',
        'Mac OS X',
        'Gecko.[\d\.]*',
        'KHTML',
        'CriOS.[\d\.]*',
        'CPU iPhone OS ([0-9_])* like Mac OS X',
        'CPU OS ([0-9_])* like Mac OS X',
        'iPod',
        'like Gecko',
        'compatible',
        'x86_..',
        'i686',
        'x64',
        'X11',
        'rv:[\d\.]*',
        'Version.[\d\.]*',
        'WOW64',
        'Win64',
        'Dalvik.[\d\.]*',
        '\.NET CLR [\d\.]*',
        'Presto.[\d\.]*',
        'Media Center PC',
    );

    protected static $crawlers = array(
        '008\\/',
        'A6-Indexer',
        'Aboundex',
        'Accoona-AI-Agent',
        'acoon',
        'AddThis',
        'ADmantX',
        'AHC',
        'Airmail',
        'Anemone',
        'Arachmo',
        'archive-com',
        'B-l-i-t-z-B-O-T',
        'bibnum\.bnf',
        'biglotron',
        'binlar',
        'boitho\.com-dc',
        'BUbiNG',
        'Butterfly\\/',
        'BuzzSumo',
        'CC Metadata Scaper',
        'Cerberian Drtrs',
        'changedetection',
        'Charlotte',
        'clips\.ua\.ac\.be',
        'CloudFlare-AlwaysOnline',
        'coccoc',
        'Commons-HttpClient',
        'convera',
        'cosmos',
        'Covario-IDS',
        'curl',
        'CyberPatrol',
        'DataparkSearch',
        'dataprovider',
        'Digg',
        'DomainAppender',
        'drupact',
        'EARTHCOM',
        'ec2linkfinder',
        'ElectricMonk',
        'Embedly',
        'europarchive\.org',
        'EventMachine HttpClient',
        'ezooms',
        'eZ Publish Link Validator',
        'facebookexternalhit',
        'Feedfetcher-Google',
        'FeedValidator',
        'FindLinks',
        'findlink',
        'findthatfile',
        'Flamingo_SearchEngine',
        'fluffy',
        'getprismatic\.com',
        'g00g1e\.net',
        'GigablastOpenSource',
        'grub-client',
        'Genieo',
        'Go-http-client',
        'Google-HTTP-Java-Client',
        'Google favicon',
        'heritrix',
        'Holmes',
        'htdig',
        'httpunit',
        'httrack',
        'ichiro',
        'igdeSpyder',
        'InAGist',
        'InfoWizards Reciprocal Link System PRO',
        'integromedb',
        'IODC',
        'IOI',
        'ips-agent',
        'iZSearch',
        'L\.webis',
        'Larbin',
        'libwww',
        'Link Valet',
        'linkdex',
        'LinkExaminer',
        'LinkWalker',
        'Lipperhey Link Explorer',
        'Lipperhey SEO Service',
        'LongURL API',
        'ltx71',
        'lwp-trivial',
        'MegaIndex\.ru',
        'mabontland',
        'MagpieRSS',
        'Mediapartners-Google',
        'MetaURI',
        'Mnogosearch',
        'mogimogi',
        'Morning Paper',
        'Mrcgiguy',
        'MVAClient',
        'netresearchserver',
        'NewsGator',
        'newsme',
        'NG-Search',
        '^NING\\/',
        'Notifixious',
        'nutch',
        'NutchCVS',
        'Nymesis',
        'oegp',
        'online link validator',
        'Online Website Link Checker',
        'Orbiter',
        'ow\.ly',
        'Ploetz \+ Zeller',
        'page2rss',
        'panscient',
        'Peew',
        'phpcrawl',
        'Pizilla',
        'Plukkie',
        'Pompos',
        'postano',
        'PostPost',
        'postrank',
        'proximic',
        'PycURL',
        'Python-httplib2',
        'python-requests',
        'Python-urllib',
        'Qseero',
        'Qwantify',
        'Radian6',
        'RebelMouse',
        'REL Link Checker',
        'RetrevoPageAnalyzer',
        'Riddler',
        'Robosourcer',
        'Ruby',
        'SBIder',
        'ScoutJet',
        'ScoutURLMonitor',
        'Scrapy',
        'Scrubby',
        'SearchSight',
        'semanticdiscovery',
        'SEOstats',
        'Seznam screenshot-generator',
        'ShopWiki',
        'SiteBar',
        'siteexplorer\.info',
        'slider\.com',
        'slurp',
        'Snappy',
        'sogou',
        'speedy',
        'Sqworm',
        'StackRambler',
        'Stratagems Kumo',
        'summify',
        'teoma',
        'theoldreader\.com',
        'TinEye',
        'Traackr.com',
        'truwoGPS',
        'tweetedtimes\.com',
        'Twikle',
        'UnwindFetchor',
        'updated',
        'urlresolver',
        'Validator\.nu\\/LV',
        'Vagabondo',
        'Vivante Link Checker',
        'Vortex',
        'voyager\\/',
        'VYU2',
        'W3C-checklink',
        'W3C_CSS_Validator_JFouffa',
        'W3C_I18n-Checker',
        'W3C-mobileOK',
        'W3C_Unicorn',
        'W3C_Validator',
        'WebIndex',
        'Websquash\.com',
        'webcollage',
        'webmon ',
        'WeSEE:Search',
        'wf84',
        'wget',
        'WomlpeFactory',
        'wotbox',
        'Xenu Link Sleuth',
        'XML Sitemaps Generator',
        'Y!J-ASR',
        'yacy',
        'Yahoo Link Preview',
        'Yahoo! Slurp China',
        'Yahoo! Slurp',
        'YahooSeeker',
        'YahooSeeker-Testing',
        'YandexImages',
        'YandexMetrika',
        'yandex',
        'yanga',
        'yeti',
        'yoogliFetchAgent',
        'Zao',
        'ZyBorg',
        '[a-z0-9\\-_]*((?<!cu)bot|crawler|archiver|transcoder|spider)',
    );

    /**
     * All possible HTTP headers that represent the
     * User-Agent string.
     *
     * @var array
     */
    protected static $uaHttpHeaders = array(
        // The default User-Agent string.
        'HTTP_USER_AGENT',
        // Header can occur on devices using Opera Mini.
        'HTTP_X_OPERAMINI_PHONE_UA',
        // Vodafone specific header: http://www.seoprinciple.com/mobile-web-community-still-angry-at-vodafone/24/

        'HTTP_X_DEVICE_USER_AGENT',
        'HTTP_X_ORIGINAL_USER_AGENT',
        'HTTP_X_SKYFIRE_PHONE',
        'HTTP_X_BOLT_PHONE_UA',
        'HTTP_DEVICE_STOCK_UA',
        'HTTP_X_UCBROWSER_DEVICE_UA',
    );

    /**
     * Class constructor.
     */
    public function __construct(array $headers = null, $userAgent = null)
    {
        $this->setHttpHeaders($headers);
        $this->setUserAgent($userAgent);
    }

    public function setHttpHeaders($httpHeaders = null)
// Bad
class Router
{
    public function generate($path)
    {
        return $_SERVER['HOST'].$path;
    }
}

// Better
class Router
{
    private $host;

    public function __construct($host)
    {
        $this->host = $host;
    }

    public function generate($path)
    {
        return $this->host.$path;
    }
}

class Controller
{
    public function myAction(Request $request)
    {
        // Instead of
        $page = isset($_GET['page']) ? intval($_GET['page']) : 1;

        // Better (assuming you use the Symfony2 request)
        $page = $request->query->get('page', 1);
    }
}
    {
        // use global _SERVER if $httpHeaders aren't defined
        if (!is_array($httpHeaders) || !count($httpHeaders)) {
            $httpHeaders = $_SERVER;
        }
        // clear existing headers
        $this->httpHeaders = array();
        // Only save HTTP headers. In PHP land, that means only _SERVER vars that
        // start with HTTP_.
        foreach ($httpHeaders as $key => $value) {
            if (substr($key, 0, 5) === 'HTTP_') {
                $this->httpHeaders[$key] = $value;
            }
        }
    }

    public function getUaHttpHeaders()
    {
        return self::$uaHttpHeaders;
    }

    public function setUserAgent($userAgent = null)
    {
        if (false === empty($userAgent)) {
            return $this->userAgent = $userAgent;
        } else {
            $this->userAgent = null;
            foreach ($this->getUaHttpHeaders() as $altHeader) {
                if (false === empty($this->httpHeaders[$altHeader])) { // @todo: should use getHttpHeader(), but it would be slow. (Serban)
                    $this->userAgent .= $this->httpHeaders[$altHeader].' ';
                }
            }

            return $this->userAgent = (!empty($this->userAgent) ? trim($this->userAgent) : null);
        }
    }

    public function getRegex()
    {
        return '('.implode('|', self::$crawlers).')';
    }

    public function getIgnored()
    {
        return '('.implode('|', self::$ignore).')';
    }

    public function isCrawler($userAgent = null)
    {
        $agent = is_null($userAgent) ? $this->userAgent : $userAgent;

        $agent = preg_replace('/'.$this->getIgnored().'/i', '', $agent);

        $result = preg_match('/'.$this->getRegex().'/i', $agent, $matches);

        if ($matches) {

            $this->matches = $matches;
        }

        return (bool) $result;
    }

    public function getMatches()
    {
        return $this->matches[0];
    }
}


1			<?php
2
3			namespace Jaybizzle\CrawlerDetect;
4
5			class CrawlerDetect
6			{
7			protected $userAgent = null;
8
9			protected $httpHeaders = array();
10
11			protected $matches = array();
12
13			/**
14			* List of strings to remove from the user agent before running the crawler regex
15			* Over a large list of user agents, this gives us about a 55% speed increase!
16			*
17			* @var array
18			*/
19			protected static $ignore = array(
20			'Safari.[\d\.]*',
21			'Firefox.[\d\.]*',
22			'Chrome.[\d\.]*',
23			'Chromium.[\d\.]*',
24			'MSIE.[\d\.]',
25			'Opera\/[\d\.]*',
26			'Mozilla.[\d\.]*',
27			'AppleWebKit.[\d\.]*',
28			'Trident.[\d\.]*',
29			'Windows NT.[\d\.]*',
30			'Android.[\d\.]*',
31			'Macintosh.',
32			'Ubuntu',
33			'Linux',
34			'Intel',
35			'Mac OS X',
36			'Gecko.[\d\.]*',
37			'KHTML',
38			'CriOS.[\d\.]*',
39			'CPU iPhone OS ([0-9_])* like Mac OS X',
40			'CPU OS ([0-9_])* like Mac OS X',
41			'iPod',
42			'like Gecko',
43			'compatible',
44			'x86_..',
45			'i686',
46			'x64',
47			'X11',
48			'rv:[\d\.]*',
49			'Version.[\d\.]*',
50			'WOW64',
51			'Win64',
52			'Dalvik.[\d\.]*',
53			'\.NET CLR [\d\.]*',
54			'Presto.[\d\.]*',
55			'Media Center PC',
56			);
57
58			protected static $crawlers = array(
59			'008\\/',
60			'A6-Indexer',
61			'Aboundex',
62			'Accoona-AI-Agent',
63			'acoon',
64			'AddThis',
65			'ADmantX',
66			'AHC',
67			'Airmail',
68			'Anemone',
69			'Arachmo',
70			'archive-com',
71			'B-l-i-t-z-B-O-T',
72			'bibnum\.bnf',
73			'biglotron',
74			'binlar',
75			'boitho\.com-dc',
76			'BUbiNG',
77			'Butterfly\\/',
78			'BuzzSumo',
79			'CC Metadata Scaper',
80			'Cerberian Drtrs',
81			'changedetection',
82			'Charlotte',
83			'clips\.ua\.ac\.be',
84			'CloudFlare-AlwaysOnline',
85			'coccoc',
86			'Commons-HttpClient',
87			'convera',
88			'cosmos',
89			'Covario-IDS',
90			'curl',
91			'CyberPatrol',
92			'DataparkSearch',
93			'dataprovider',
94			'Digg',
95			'DomainAppender',
96			'drupact',
97			'EARTHCOM',
98			'ec2linkfinder',
99			'ElectricMonk',
100			'Embedly',
101			'europarchive\.org',
102			'EventMachine HttpClient',
103			'ezooms',
104			'eZ Publish Link Validator',
105			'facebookexternalhit',
106			'Feedfetcher-Google',
107			'FeedValidator',
108			'FindLinks',
109			'findlink',
110			'findthatfile',
111			'Flamingo_SearchEngine',
112			'fluffy',
113			'getprismatic\.com',
114			'g00g1e\.net',
115			'GigablastOpenSource',
116			'grub-client',
117			'Genieo',
118			'Go-http-client',
119			'Google-HTTP-Java-Client',
120			'Google favicon',
121			'heritrix',
122			'Holmes',
123			'htdig',
124			'httpunit',
125			'httrack',
126			'ichiro',
127			'igdeSpyder',
128			'InAGist',
129			'InfoWizards Reciprocal Link System PRO',
130			'integromedb',
131			'IODC',
132			'IOI',
133			'ips-agent',
134			'iZSearch',
135			'L\.webis',
136			'Larbin',
137			'libwww',
138			'Link Valet',
139			'linkdex',
140			'LinkExaminer',
141			'LinkWalker',
142			'Lipperhey Link Explorer',
143			'Lipperhey SEO Service',
144			'LongURL API',
145			'ltx71',
146			'lwp-trivial',
147			'MegaIndex\.ru',
148			'mabontland',
149			'MagpieRSS',
150			'Mediapartners-Google',
151			'MetaURI',
152			'Mnogosearch',
153			'mogimogi',
154			'Morning Paper',
155			'Mrcgiguy',
156			'MVAClient',
157			'netresearchserver',
158			'NewsGator',
159			'newsme',
160			'NG-Search',
161			'^NING\\/',
162			'Notifixious',
163			'nutch',
164			'NutchCVS',
165			'Nymesis',
166			'oegp',
167			'online link validator',
168			'Online Website Link Checker',
169			'Orbiter',
170			'ow\.ly',
171			'Ploetz \+ Zeller',
172			'page2rss',
173			'panscient',
174			'Peew',
175			'phpcrawl',
176			'Pizilla',
177			'Plukkie',
178			'Pompos',
179			'postano',
180			'PostPost',
181			'postrank',
182			'proximic',
183			'PycURL',
184			'Python-httplib2',
185			'python-requests',
186			'Python-urllib',
187			'Qseero',
188			'Qwantify',
189			'Radian6',
190			'RebelMouse',
191			'REL Link Checker',
192			'RetrevoPageAnalyzer',
193			'Riddler',
194			'Robosourcer',
195			'Ruby',
196			'SBIder',
197			'ScoutJet',
198			'ScoutURLMonitor',
199			'Scrapy',
200			'Scrubby',
201			'SearchSight',
202			'semanticdiscovery',
203			'SEOstats',
204			'Seznam screenshot-generator',
205			'ShopWiki',
206			'SiteBar',
207			'siteexplorer\.info',
208			'slider\.com',
209			'slurp',
210			'Snappy',
211			'sogou',
212			'speedy',
213			'Sqworm',
214			'StackRambler',
215			'Stratagems Kumo',
216			'summify',
217			'teoma',
218			'theoldreader\.com',
219			'TinEye',
220			'Traackr.com',
221			'truwoGPS',
222			'tweetedtimes\.com',
223			'Twikle',
224			'UnwindFetchor',
225			'updated',
226			'urlresolver',
227			'Validator\.nu\\/LV',
228			'Vagabondo',
229			'Vivante Link Checker',
230			'Vortex',
231			'voyager\\/',
232			'VYU2',
233			'W3C-checklink',
234			'W3C_CSS_Validator_JFouffa',
235			'W3C_I18n-Checker',
236			'W3C-mobileOK',
237			'W3C_Unicorn',
238			'W3C_Validator',
239			'WebIndex',
240			'Websquash\.com',
241			'webcollage',
242			'webmon ',
243			'WeSEE:Search',
244			'wf84',
245			'wget',
246			'WomlpeFactory',
247			'wotbox',
248			'Xenu Link Sleuth',
249			'XML Sitemaps Generator',
250			'Y!J-ASR',
251			'yacy',
252			'Yahoo Link Preview',
253			'Yahoo! Slurp China',
254			'Yahoo! Slurp',
255			'YahooSeeker',
256			'YahooSeeker-Testing',
257			'YandexImages',
258			'YandexMetrika',
259			'yandex',
260			'yanga',
261			'yeti',
262			'yoogliFetchAgent',
263			'Zao',
264			'ZyBorg',
265			'[a-z0-9\\-_]*((?<!cu)bot\|crawler\|archiver\|transcoder\|spider)',
266			);
267
268			/**
269			* All possible HTTP headers that represent the
270			* User-Agent string.
271			*
272			* @var array
273			*/
274			protected static $uaHttpHeaders = array(
275			// The default User-Agent string.
276			'HTTP_USER_AGENT',
277			// Header can occur on devices using Opera Mini.
278			'HTTP_X_OPERAMINI_PHONE_UA',
279			// Vodafone specific header: http://www.seoprinciple.com/mobile-web-community-still-angry-at-vodafone/24/
			0 ignored issues – show Unused Code Comprehensibility introduced 2015-11-25 13:32 UTC by Report Bug Copy Issue Report `38%` of this comment could be valid code. Did you maybe forget this after debugging? Sometimes obsolete code just ends up commented out instead of removed. In this case it is better to remove the code once you have checked you do not need it. The code might also have been commented out for debugging purposes. In this case it is vital that someone uncomments it again or your project may behave in very unexpected ways in production. This check looks for comments that seem to be mostly valid code and reports them. Loading history...
280			'HTTP_X_DEVICE_USER_AGENT',
281			'HTTP_X_ORIGINAL_USER_AGENT',
282			'HTTP_X_SKYFIRE_PHONE',
283			'HTTP_X_BOLT_PHONE_UA',
284			'HTTP_DEVICE_STOCK_UA',
285			'HTTP_X_UCBROWSER_DEVICE_UA',
286			);
287
288			/**
289			* Class constructor.
290			*/
291			public function __construct(array $headers = null, $userAgent = null)
292			{
293			$this->setHttpHeaders($headers);
294			$this->setUserAgent($userAgent);
295			}
296
297			public function setHttpHeaders($httpHeaders = null)
			0 ignored issues – show Coding Style introduced 2015-11-25 13:32 UTC by Report Bug Copy Issue Report `setHttpHeaders` uses the super-global variable `$_SERVER` which is generally not recommended. Instead of super-globals, we recommend to explicitly inject the dependencies of your class. This makes your code less dependent on global state and it becomes generally more testable: // Bad class Router { public function generate($path) { return $_SERVER['HOST'].$path; } } // Better class Router { private $host; public function __construct($host) { $this->host = $host; } public function generate($path) { return $this->host.$path; } } class Controller { public function myAction(Request $request) { // Instead of $page = isset($_GET['page']) ? intval($_GET['page']) : 1; // Better (assuming you use the Symfony2 request) $page = $request->query->get('page', 1); } } Loading history...
298			{
299			// use global _SERVER if $httpHeaders aren't defined
300			if (!is_array($httpHeaders) \|\| !count($httpHeaders)) {
301			$httpHeaders = $_SERVER;
302			}
303			// clear existing headers
304			$this->httpHeaders = array();
305			// Only save HTTP headers. In PHP land, that means only _SERVER vars that
306			// start with HTTP_.
307			foreach ($httpHeaders as $key => $value) {
308			if (substr($key, 0, 5) === 'HTTP_') {
309			$this->httpHeaders[$key] = $value;
310			}
311			}
312			}
313
314			public function getUaHttpHeaders()
315			{
316			return self::$uaHttpHeaders;
317			}
318
319			public function setUserAgent($userAgent = null)
320			{
321			if (false === empty($userAgent)) {
322			return $this->userAgent = $userAgent;
323			} else {
324			$this->userAgent = null;
325			foreach ($this->getUaHttpHeaders() as $altHeader) {
326			if (false === empty($this->httpHeaders[$altHeader])) { // @todo: should use getHttpHeader(), but it would be slow. (Serban)
327			$this->userAgent .= $this->httpHeaders[$altHeader].' ';
328			}
329			}
330
331			return $this->userAgent = (!empty($this->userAgent) ? trim($this->userAgent) : null);
332			}
333			}
334
335			public function getRegex()
336			{
337			return '('.implode('\|', self::$crawlers).')';
338			}
339
340			public function getIgnored()
341			{
342			return '('.implode('\|', self::$ignore).')';
343			}
344
345			public function isCrawler($userAgent = null)
346			{
347			$agent = is_null($userAgent) ? $this->userAgent : $userAgent;
348
349			$agent = preg_replace('/'.$this->getIgnored().'/i', '', $agent);
350
351			$result = preg_match('/'.$this->getRegex().'/i', $agent, $matches);
352
353			if ($matches) {
			0 ignored issues – show Bug Best Practice introduced 2015-11-25 13:32 UTC by Report Bug Copy Issue Report The expression `$matches` of type `string[]` is implicitly converted to a boolean; are you sure this is intended? If so, consider using `! empty($expr)` instead to make it clear that you intend to check for an array without elements. This check marks implicit conversions of arrays to boolean values in a comparison. While in PHP an empty array is considered to be equal (but not identical) to false, this is not always apparent. Consider making the comparison explicit by using `empty(..)` or `! empty(...)` instead. Loading history...
354			$this->matches = $matches;
355			}
356
357			return (bool) $result;
358			}
359
360			public function getMatches()
361			{
362			return $this->matches[0];
363			}
364			}
365

JayBizzle / Crawler-Detect

Pull Request — master (#42)

CrawlerDetect::getRegex() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like