1 | <?php |
||
14 | class Crawlers extends AbstractProvider |
||
15 | { |
||
16 | /** |
||
17 | * Array of regular expressions to match against the user agent. |
||
18 | * |
||
19 | * @var array |
||
20 | */ |
||
21 | protected $data = array( |
||
22 | ' YLT', |
||
23 | '^b0t$', |
||
24 | '^bluefish ', |
||
25 | '^Calypso v\/', |
||
26 | '^Corax', |
||
27 | '^COMODO DCV', |
||
28 | '^DangDang', |
||
29 | '^DavClnt', |
||
30 | '^DHSH', |
||
31 | '^Expanse', |
||
32 | '^FDM ', |
||
33 | '^git\/', |
||
34 | '^Goose\/', |
||
35 | '^Grabber', |
||
36 | '^HTTPClient\/', |
||
37 | '^HTTPing', |
||
38 | '^Java\/', |
||
39 | '^Jeode\/', |
||
40 | '^Jetty\/', |
||
41 | '^Mail\/', |
||
42 | '^Mget', |
||
43 | '^Microsoft URL Control', |
||
44 | '^Mikrotik\/', |
||
45 | '^Netlab360', |
||
46 | '^NG\/[0-9\.]', |
||
47 | '^NING\/', |
||
48 | '^Nuclei', |
||
49 | '^PHP\/', |
||
50 | '^RMA\/', |
||
51 | '^Ruby|Ruby\/[0-9]', |
||
52 | '^Swurl ', |
||
53 | '^TLS tester ', |
||
54 | '^VSE\/[0-9]', |
||
55 | '^WordPress\.com', |
||
56 | '^XRL\/[0-9]', |
||
57 | '^ZmEu', |
||
58 | '008\/', |
||
59 | '13TABS', |
||
60 | '192\.comAgent', |
||
61 | '2GDPR\/', |
||
62 | '2ip\.ru', |
||
63 | '404enemy', |
||
64 | '7Siters', |
||
65 | '80legs', |
||
66 | 'a\.pr-cy\.ru', |
||
67 | 'a3logics\.in', |
||
68 | 'A6-Indexer', |
||
69 | 'Abonti', |
||
70 | 'Aboundex', |
||
71 | 'aboutthedomain', |
||
72 | 'Accoona-AI-Agent', |
||
73 | 'acebookexternalhit\/', |
||
74 | 'acoon', |
||
75 | 'acrylicapps\.com\/pulp', |
||
76 | 'Acunetix', |
||
77 | 'AdAuth\/', |
||
78 | 'adbeat', |
||
79 | 'AddThis', |
||
80 | 'ADmantX', |
||
81 | 'AdminLabs', |
||
82 | 'adressendeutschland', |
||
83 | 'adreview\/', |
||
84 | 'adscanner', |
||
85 | 'Adstxtaggregator', |
||
86 | 'adstxt-worker', |
||
87 | 'adstxt\.com', |
||
88 | 'AffiliateLabz\/', |
||
89 | 'agentslug', |
||
90 | 'AHC', |
||
91 | 'aihit', |
||
92 | 'aiohttp\/', |
||
93 | 'Airmail', |
||
94 | 'akka-http\/', |
||
95 | 'akula\/', |
||
96 | 'alertra', |
||
97 | 'alexa site audit', |
||
98 | 'Alibaba\.Security\.Heimdall', |
||
99 | 'Alligator', |
||
100 | 'allloadin', |
||
101 | 'AllSubmitter', |
||
102 | 'alyze\.info', |
||
103 | 'amagit', |
||
104 | '^Amazon Simple Notification Service Agent$', |
||
105 | 'Anarchie', |
||
106 | 'AndroidDownloadManager', |
||
107 | 'Anemone', |
||
108 | 'AngleSharp', |
||
109 | 'annotate_google', |
||
110 | 'Anthill', |
||
111 | 'Ant\.com', |
||
112 | 'Anturis Agent', |
||
113 | 'AnyEvent-HTTP\/', |
||
114 | 'Apache Droid', |
||
115 | 'Apache OpenOffice', |
||
116 | 'Apache-HttpAsyncClient', |
||
117 | 'Apache-HttpClient', |
||
118 | 'ApacheBench', |
||
119 | 'Apexoo', |
||
120 | 'apimon\.de', |
||
121 | 'APIs-Google', |
||
122 | 'AportWorm\/', |
||
123 | 'AppBeat\/', |
||
124 | 'AppEngine-Google', |
||
125 | 'AppleSyndication', |
||
126 | 'Aprc\/[0-9]', |
||
127 | 'Arachmo', |
||
128 | 'arachnode', |
||
129 | 'Arachnophilia', |
||
130 | 'aria2', |
||
131 | 'Arukereso', |
||
132 | 'asafaweb', |
||
133 | 'Asana\/', |
||
134 | 'AskQuickly', |
||
135 | 'Ask Jeeves', |
||
136 | 'ASPSeek', |
||
137 | 'Asterias', |
||
138 | 'Astute', |
||
139 | 'asynchttp', |
||
140 | 'Attach', |
||
141 | 'attohttpc', |
||
142 | 'autocite', |
||
143 | 'AutomaticWPTester', |
||
144 | 'Autonomy', |
||
145 | 'axios\/', |
||
146 | 'AWS Security Scanner', |
||
147 | 'B-l-i-t-z-B-O-T', |
||
148 | 'Backlink-Ceck', |
||
149 | 'backlink-check', |
||
150 | 'BacklinkHttpStatus', |
||
151 | 'BackStreet', |
||
152 | 'BackupLand', |
||
153 | 'BackWeb', |
||
154 | 'Bad-Neighborhood', |
||
155 | 'Badass', |
||
156 | 'baidu\.com', |
||
157 | 'Bandit', |
||
158 | 'basicstate', |
||
159 | 'BatchFTP', |
||
160 | 'Battleztar Bazinga', |
||
161 | 'baypup\/', |
||
162 | 'BazQux', |
||
163 | 'BBBike', |
||
164 | 'BCKLINKS', |
||
165 | 'BDFetch', |
||
166 | 'BegunAdvertising', |
||
167 | 'Bewica-security-scan', |
||
168 | 'Bidtellect', |
||
169 | 'BigBozz', |
||
170 | 'Bigfoot', |
||
171 | 'biglotron', |
||
172 | 'BingLocalSearch', |
||
173 | 'BingPreview', |
||
174 | 'binlar', |
||
175 | 'biNu image cacher', |
||
176 | 'Bitacle', |
||
177 | 'Bitrix link preview', |
||
178 | 'biz_Directory', |
||
179 | 'BKCTwitterUnshortener\/', |
||
180 | 'Black Hole', |
||
181 | 'Blackboard Safeassign', |
||
182 | 'BlackWidow', |
||
183 | 'BlockNote\.Net', |
||
184 | 'BlogBridge', |
||
185 | 'Bloglines', |
||
186 | 'Bloglovin', |
||
187 | 'BlogPulseLive', |
||
188 | 'BlogSearch', |
||
189 | 'Blogtrottr', |
||
190 | 'BlowFish', |
||
191 | 'boitho\.com-dc', |
||
192 | 'Boost\.Beast', |
||
193 | 'BPImageWalker', |
||
194 | 'Braintree-Webhooks', |
||
195 | 'Branch Metrics API', |
||
196 | 'Branch-Passthrough', |
||
197 | 'Brandprotect', |
||
198 | 'BrandVerity', |
||
199 | 'Brandwatch', |
||
200 | 'Brodie\/', |
||
201 | 'Browsershots', |
||
202 | 'BUbiNG', |
||
203 | 'Buck\/', |
||
204 | 'Buddy', |
||
205 | 'BuiltWith', |
||
206 | 'Bullseye', |
||
207 | 'BunnySlippers', |
||
208 | 'Burf Search', |
||
209 | 'Butterfly\/', |
||
210 | 'BuzzSumo', |
||
211 | 'CAAM\/[0-9]', |
||
212 | 'CakePHP', |
||
213 | 'Calculon', |
||
214 | 'Canary%20Mail', |
||
215 | 'CaretNail', |
||
216 | 'catexplorador', |
||
217 | 'CC Metadata Scaper', |
||
218 | 'Cegbfeieh', |
||
219 | 'censys', |
||
220 | 'centuryb.o.t9[at]gmail.com', |
||
221 | 'Cerberian Drtrs', |
||
222 | 'CERT\.at-Statistics-Survey', |
||
223 | 'cf-facebook', |
||
224 | 'cg-eye', |
||
225 | 'changedetection', |
||
226 | 'ChangesMeter', |
||
227 | 'Charlotte', |
||
228 | 'CheckHost', |
||
229 | 'checkprivacy', |
||
230 | 'CherryPicker', |
||
231 | 'ChinaClaw', |
||
232 | 'Chirp\/', |
||
233 | 'chkme\.com', |
||
234 | 'Chlooe', |
||
235 | 'Chromaxa', |
||
236 | 'CirrusExplorer', |
||
237 | 'CISPA Web Analyser', |
||
238 | 'CISPA Vulnerability Notification', |
||
239 | 'Citoid', |
||
240 | 'CJNetworkQuality', |
||
241 | 'Clarsentia', |
||
242 | 'clips\.ua\.ac\.be', |
||
243 | 'Cloud mapping', |
||
244 | 'CloudEndure', |
||
245 | 'CloudFlare-AlwaysOnline', |
||
246 | 'Cloudflare-Healthchecks', |
||
247 | 'Cloudinary', |
||
248 | 'cmcm\.com', |
||
249 | 'coccoc', |
||
250 | 'cognitiveseo', |
||
251 | 'colly -', |
||
252 | 'CommaFeed', |
||
253 | 'Commons-HttpClient', |
||
254 | 'commonscan', |
||
255 | 'contactbigdatafr', |
||
256 | 'contentkingapp', |
||
257 | 'Contextual Code Sites Explorer', |
||
258 | 'convera', |
||
259 | 'CookieReports', |
||
260 | 'copyright sheriff', |
||
261 | 'CopyRightCheck', |
||
262 | 'Copyscape', |
||
263 | 'cortex\/', |
||
264 | 'Cosmos4j\.feedback', |
||
265 | 'Covario-IDS', |
||
266 | 'Craw\/', |
||
267 | 'Crescent', |
||
268 | 'Crowsnest', |
||
269 | 'Criteo', |
||
270 | 'CSHttp', |
||
271 | 'CSSCheck', |
||
272 | 'Cula\/', |
||
273 | 'curb', |
||
274 | 'Curious George', |
||
275 | 'curl', |
||
276 | 'cuwhois\/', |
||
277 | 'cybo\.com', |
||
278 | 'DAP\/NetHTTP', |
||
279 | 'DareBoost', |
||
280 | 'DatabaseDriverMysqli', |
||
281 | 'DataCha0s', |
||
282 | 'Datafeedwatch', |
||
283 | 'Datanyze', |
||
284 | 'DataparkSearch', |
||
285 | 'dataprovider', |
||
286 | 'DataXu', |
||
287 | 'Daum(oa)?[ \/][0-9]', |
||
288 | 'dBpoweramp', |
||
289 | 'ddline', |
||
290 | 'deeris', |
||
291 | 'delve\.ai', |
||
292 | 'Demon', |
||
293 | 'DeuSu', |
||
294 | 'developers\.google\.com\/\+\/web\/snippet\/', |
||
295 | 'Devil', |
||
296 | 'Digg', |
||
297 | 'Digincore', |
||
298 | 'DigitalPebble', |
||
299 | 'Dirbuster', |
||
300 | 'Discourse Forum Onebox', |
||
301 | 'Disqus\/', |
||
302 | 'Dispatch\/', |
||
303 | 'DittoSpyder', |
||
304 | 'dlvr', |
||
305 | 'DMBrowser', |
||
306 | 'DNSPod-reporting', |
||
307 | 'docoloc', |
||
308 | 'Dolphin http client', |
||
309 | 'DomainAppender', |
||
310 | 'DomainLabz', |
||
311 | 'Domains Project\/', |
||
312 | 'Donuts Content Explorer', |
||
313 | 'dotMailer content retrieval', |
||
314 | 'dotSemantic', |
||
315 | 'downforeveryoneorjustme', |
||
316 | 'Download Wonder', |
||
317 | 'downnotifier', |
||
318 | 'DowntimeDetector', |
||
319 | 'Drip', |
||
320 | 'drupact', |
||
321 | 'Drupal \(\+http:\/\/drupal\.org\/\)', |
||
322 | 'DTS Agent', |
||
323 | 'dubaiindex', |
||
324 | 'DuplexWeb-Google', |
||
325 | 'DynatraceSynthetic', |
||
326 | 'EARTHCOM', |
||
327 | 'Easy-Thumb', |
||
328 | 'EasyDL', |
||
329 | 'Ebingbong', |
||
330 | 'ec2linkfinder', |
||
331 | 'eCairn-Grabber', |
||
332 | 'eCatch', |
||
333 | 'ECCP', |
||
334 | 'eContext\/', |
||
335 | 'Ecxi', |
||
336 | 'EirGrabber', |
||
337 | 'ElectricMonk', |
||
338 | 'elefent', |
||
339 | 'EMail Exractor', |
||
340 | 'EMail Wolf', |
||
341 | 'EmailWolf', |
||
342 | 'Embarcadero', |
||
343 | 'Embed PHP Library', |
||
344 | 'Embedly', |
||
345 | 'endo\/', |
||
346 | 'europarchive\.org', |
||
347 | 'evc-batch', |
||
348 | 'EventMachine HttpClient', |
||
349 | 'Everwall Link Expander', |
||
350 | 'Evidon', |
||
351 | 'Evrinid', |
||
352 | 'ExactSearch', |
||
353 | 'ExaleadCloudview', |
||
354 | 'Excel\/', |
||
355 | 'exif', |
||
356 | 'ExoRank', |
||
357 | 'Exploratodo', |
||
358 | 'Express WebPictures', |
||
359 | 'Extreme Picture Finder', |
||
360 | 'EyeNetIE', |
||
361 | 'ezooms', |
||
362 | 'facebookexternalhit', |
||
363 | 'facebookexternalua', |
||
364 | 'facebookplatform', |
||
365 | 'fairshare', |
||
366 | 'Faraday v', |
||
367 | 'fasthttp', |
||
368 | 'Faveeo', |
||
369 | 'Favicon downloader', |
||
370 | 'faviconkit', |
||
371 | 'faviconarchive', |
||
372 | 'FavOrg', |
||
373 | 'Feed Wrangler', |
||
374 | 'Feedable\/', |
||
375 | 'Feedbin', |
||
376 | 'FeedBooster', |
||
377 | 'FeedBucket', |
||
378 | 'FeedBunch\/', |
||
379 | 'FeedBurner', |
||
380 | 'feeder', |
||
381 | 'Feedly', |
||
382 | 'Feedshow\/', |
||
383 | 'FeedshowOnline', |
||
384 | 'Feedspot', |
||
385 | 'FeedViewer\/', |
||
386 | 'Feedwind\/', |
||
387 | 'FeedZcollector', |
||
388 | 'feeltiptop', |
||
389 | 'Fetch API', |
||
390 | 'Fetch\/[0-9]', |
||
391 | 'Fever\/[0-9]', |
||
392 | 'FHscan', |
||
393 | 'Fiery%20Feeds', |
||
394 | 'Filestack', |
||
395 | 'Fimap', |
||
396 | 'findlink', |
||
397 | 'findthatfile', |
||
398 | 'FlashGet', |
||
399 | 'FlipboardBrowserProxy', |
||
400 | 'FlipboardProxy', |
||
401 | 'FlipboardRSS', |
||
402 | 'Flock\/', |
||
403 | 'Florienzh\/', |
||
404 | 'fluffy', |
||
405 | 'Flunky', |
||
406 | 'flynxapp', |
||
407 | 'forensiq', |
||
408 | 'FoundSeoTool', |
||
409 | 'http:\/\/www.neomo.de\/', //'Francis [Bot]' |
||
410 | 'free thumbnails', |
||
411 | 'Freeuploader', |
||
412 | 'FreshRSS', |
||
413 | 'Funnelback', |
||
414 | 'Fuzz Faster U Fool', |
||
415 | 'G-i-g-a-b-o-t', |
||
416 | 'g00g1e\.net', |
||
417 | 'ganarvisitas', |
||
418 | 'gdnplus\.com', |
||
419 | 'geek-tools', |
||
420 | 'Genieo', |
||
421 | 'GentleSource', |
||
422 | 'GetCode', |
||
423 | 'Getintent', |
||
424 | 'GetLinkInfo', |
||
425 | 'getprismatic', |
||
426 | 'GetRight', |
||
427 | 'getroot', |
||
428 | 'GetURLInfo\/', |
||
429 | 'GetWeb', |
||
430 | 'Geziyor', |
||
431 | 'Ghost Inspector', |
||
432 | 'GigablastOpenSource', |
||
433 | 'GIS-LABS', |
||
434 | 'github-camo', |
||
435 | 'GitHub-Hookshot', |
||
436 | 'github\.com', |
||
437 | 'Goldfire Server', |
||
438 | 'Go [\d\.]* package http', |
||
439 | 'Go http package', |
||
440 | 'Go-Ahead-Got-It', |
||
441 | 'Go-http-client', |
||
442 | 'go-mtasts\/', |
||
443 | 'Go!Zilla', |
||
444 | 'gobyus', |
||
445 | 'Gofeed', |
||
446 | 'gofetch', |
||
447 | 'GomezAgent', |
||
448 | 'gooblog', |
||
449 | 'Goodzer\/', |
||
450 | 'Google AppsViewer', |
||
451 | 'Google Desktop', |
||
452 | 'Google favicon', |
||
453 | 'Google Keyword Suggestion', |
||
454 | 'Google Keyword Tool', |
||
455 | 'Google Page Speed Insights', |
||
456 | 'Google-Podcast', |
||
457 | 'Google PP Default', |
||
458 | 'Google Search Console', |
||
459 | 'Google Web Preview', |
||
460 | 'Google-Ads-Creatives-Assistant', |
||
461 | 'Google-Ads-Overview', |
||
462 | 'Google-Adwords', |
||
463 | 'Google-Apps-Script', |
||
464 | 'Google-Calendar-Importer', |
||
465 | 'Google-HotelAdsVerifier', |
||
466 | 'Google-HTTP-Java-Client', |
||
467 | 'Google-SMTP-STS', |
||
468 | 'Google-Publisher-Plugin', |
||
469 | 'Google-Read-Aloud', |
||
470 | 'Google-SearchByImage', |
||
471 | 'Google-Site-Verification', |
||
472 | 'Google-speakr', |
||
473 | 'Google-Structured-Data-Testing-Tool', |
||
474 | 'Google-Youtube-Links', |
||
475 | 'google-xrawler', |
||
476 | 'GoogleDocs', |
||
477 | 'GoogleHC\/', |
||
478 | 'GoogleProducer', |
||
479 | 'GoogleSites', |
||
480 | 'Google-Transparency-Report', |
||
481 | 'Gookey', |
||
482 | 'GoSpotCheck', |
||
483 | 'gosquared-thumbnailer', |
||
484 | 'Gotit', |
||
485 | 'GoZilla', |
||
486 | 'grabify', |
||
487 | 'GrabNet', |
||
488 | 'Grafula', |
||
489 | 'Grammarly', |
||
490 | 'GrapeFX', |
||
491 | 'GreatNews', |
||
492 | 'Gregarius', |
||
493 | 'GRequests', |
||
494 | 'grokkit', |
||
495 | 'grouphigh', |
||
496 | 'grub-client', |
||
497 | 'gSOAP\/', |
||
498 | 'GT::WWW', |
||
499 | 'GTmetrix', |
||
500 | 'GuzzleHttp', |
||
501 | 'gvfs\/', |
||
502 | 'HAA(A)?RTLAND http client', |
||
503 | 'Haansoft', |
||
504 | 'hackney\/', |
||
505 | 'Hadi Agent', |
||
506 | 'HappyApps-WebCheck', |
||
507 | 'Hardenize', |
||
508 | 'Hatena', |
||
509 | 'Havij', |
||
510 | 'HaxerMen', |
||
511 | 'HeadlessChrome', |
||
512 | 'HEADMasterSEO', |
||
513 | 'HeartRails_Capture', |
||
514 | 'help@dataminr\.com', |
||
515 | 'heritrix', |
||
516 | 'Hexometer', |
||
517 | 'historious', |
||
518 | 'hkedcity', |
||
519 | 'hledejLevne\.cz', |
||
520 | 'Hloader', |
||
521 | 'HMView', |
||
522 | 'Holmes', |
||
523 | 'HonesoSearchEngine', |
||
524 | 'HootSuite Image proxy', |
||
525 | 'Hootsuite-WebFeed', |
||
526 | 'hosterstats', |
||
527 | 'HostTracker', |
||
528 | 'ht:\/\/check', |
||
529 | 'htdig', |
||
530 | 'HTMLparser', |
||
531 | 'htmlyse', |
||
532 | 'HTTP Banner Detection', |
||
533 | 'HTTP_Compression_Test', |
||
534 | 'http_request2', |
||
535 | 'http_requester', |
||
536 | 'http-get', |
||
537 | 'HTTP-Header-Abfrage', |
||
538 | 'http-kit', |
||
539 | 'http-request\/', |
||
540 | 'HTTP-Tiny', |
||
541 | 'HTTP::Lite', |
||
542 | 'http\.rb\/', |
||
543 | 'http_get', |
||
544 | 'HttpComponents', |
||
545 | 'httphr', |
||
546 | 'HTTPMon', |
||
547 | 'HTTPie', |
||
548 | 'httpRequest', |
||
549 | 'httpscheck', |
||
550 | 'httpssites_power', |
||
551 | 'httpunit', |
||
552 | 'HttpUrlConnection', |
||
553 | 'httrack', |
||
554 | 'huaweisymantec', |
||
555 | 'HubSpot ', |
||
556 | 'HubSpot-Link-Resolver', |
||
557 | 'Humanlinks', |
||
558 | 'i2kconnect\/', |
||
559 | 'Iblog', |
||
560 | 'ichiro', |
||
561 | 'Id-search', |
||
562 | 'IdeelaborPlagiaat', |
||
563 | 'IDG Twitter Links Resolver', |
||
564 | 'IDwhois\/', |
||
565 | 'Iframely', |
||
566 | 'igdeSpyder', |
||
567 | 'iGooglePortal', |
||
568 | 'IlTrovatore', |
||
569 | 'Image Fetch', |
||
570 | 'Image Sucker', |
||
571 | 'ImageEngine\/', |
||
572 | 'ImageVisu\/', |
||
573 | 'Imagga', |
||
574 | 'imagineeasy', |
||
575 | 'imgsizer', |
||
576 | 'InAGist', |
||
577 | 'inbound\.li parser', |
||
578 | 'InDesign%20CC', |
||
579 | 'Indy Library', |
||
580 | 'InetURL', |
||
581 | 'infegy', |
||
582 | 'infohelfer', |
||
583 | 'InfoTekies', |
||
584 | 'InfoWizards Reciprocal Link', |
||
585 | 'inpwrd\.com', |
||
586 | 'instabid', |
||
587 | 'Instapaper', |
||
588 | 'Integrity', |
||
589 | 'integromedb', |
||
590 | 'Intelliseek', |
||
591 | 'InterGET', |
||
592 | 'internet_archive', |
||
593 | 'Internet Ninja', |
||
594 | 'InternetSeer', |
||
595 | 'internetVista monitor', |
||
596 | 'internetwache', |
||
597 | 'intraVnews', |
||
598 | 'IODC', |
||
599 | 'IOI', |
||
600 | 'iplabel', |
||
601 | 'ips-agent', |
||
602 | 'IPS\/[0-9]', |
||
603 | 'IPWorks HTTP\/S Component', |
||
604 | 'iqdb\/', |
||
605 | 'Iria', |
||
606 | 'Irokez', |
||
607 | 'isitup\.org', |
||
608 | 'iskanie', |
||
609 | 'isUp\.li', |
||
610 | 'iThemes Sync\/', |
||
611 | 'IZaBEE', |
||
612 | 'iZSearch', |
||
613 | 'JAHHO', |
||
614 | 'janforman', |
||
615 | 'Jaunt\/', |
||
616 | 'Java.*outbrain', |
||
617 | 'javelin\.io', |
||
618 | 'Jbrofuzz', |
||
619 | 'Jersey\/', |
||
620 | 'JetCar', |
||
621 | 'Jigsaw', |
||
622 | 'Jobboerse', |
||
623 | 'JobFeed discovery', |
||
624 | 'Jobg8 URL Monitor', |
||
625 | 'jobo', |
||
626 | 'Jobrapido', |
||
627 | 'Jobsearch1\.5', |
||
628 | 'JoinVision Generic', |
||
629 | 'JolokiaPwn', |
||
630 | 'Joomla', |
||
631 | 'Jorgee', |
||
632 | 'JS-Kit', |
||
633 | 'JungleKeyThumbnail', |
||
634 | 'JustView', |
||
635 | 'Kaspersky Lab CFR link resolver', |
||
636 | 'Kelny\/', |
||
637 | 'Kerrigan\/', |
||
638 | 'KeyCDN', |
||
639 | 'Keyword Density', |
||
640 | 'Keywords Research', |
||
641 | 'khttp\/', |
||
642 | 'KickFire', |
||
643 | 'KimonoLabs\/', |
||
644 | 'Kml-Google', |
||
645 | 'knows\.is', |
||
646 | 'KOCMOHABT', |
||
647 | 'kouio', |
||
648 | 'kubectl', |
||
649 | 'kube-probe', |
||
650 | 'kulturarw3', |
||
651 | 'KumKie', |
||
652 | 'L\.webis', |
||
653 | 'Larbin', |
||
654 | 'Lavf\/', |
||
655 | 'leakix\.net', |
||
656 | 'LeechFTP', |
||
657 | 'LeechGet', |
||
658 | 'letsencrypt', |
||
659 | 'Lftp', |
||
660 | 'LibVLC', |
||
661 | 'LibWeb', |
||
662 | 'Libwhisker', |
||
663 | 'libwww', |
||
664 | 'Licorne', |
||
665 | 'Liferea\/', |
||
666 | 'Lightspeedsystems', |
||
667 | 'Lighthouse', |
||
668 | 'Likse', |
||
669 | 'limber\.io', |
||
670 | 'Link Valet', |
||
671 | 'link_thumbnailer', |
||
672 | 'LinkAnalyser', |
||
673 | 'LinkAlarm\/', |
||
674 | 'linkCheck', |
||
675 | 'linkdex', |
||
676 | 'LinkExaminer', |
||
677 | 'linkfluence', |
||
678 | 'linkpeek', |
||
679 | 'LinkScan', |
||
680 | 'LinksManager', |
||
681 | 'LinkPreview', |
||
682 | 'LinkTiger', |
||
683 | 'LinkWalker', |
||
684 | 'Lipperhey', |
||
685 | 'Litemage_walker', |
||
686 | 'livedoor ScreenShot', |
||
687 | 'LoadImpactRload', |
||
688 | 'localsearch-web', |
||
689 | 'LongURL API', |
||
690 | 'longurl-r-package', |
||
691 | 'looid\.com', |
||
692 | 'looksystems\.net', |
||
693 | 'ltx71', |
||
694 | 'lua-resty-http', |
||
695 | 'Lush Http Client', |
||
696 | 'lwp-request', |
||
697 | 'lwp-trivial', |
||
698 | 'LWP::Simple', |
||
699 | 'lycos', |
||
700 | 'LYT\.SR', |
||
701 | 'mabontland', |
||
702 | 'MacOutlook\/', |
||
703 | 'Mag-Net', |
||
704 | 'MagpieRSS', |
||
705 | 'Mail\.Ru', |
||
706 | 'Mail::STS', |
||
707 | 'MailChimp', |
||
708 | 'Majestic12', |
||
709 | 'makecontact\/', |
||
710 | 'Mandrill', |
||
711 | 'MapperCmd', |
||
712 | 'marketinggrader', |
||
713 | 'MarkMonitor', |
||
714 | 'MarkWatch', |
||
715 | 'Mass Downloader', |
||
716 | 'masscan\/', |
||
717 | 'Mata Hari', |
||
718 | 'mattermost', |
||
719 | 'Mediametric', |
||
720 | 'Mediapartners-Google', |
||
721 | 'mediawords', |
||
722 | 'MegaIndex\.ru', |
||
723 | 'MeltwaterNews', |
||
724 | 'Melvil Rawi', |
||
725 | 'MemGator', |
||
726 | 'Metaspinner', |
||
727 | 'MetaURI', |
||
728 | 'MFC_Tear_Sample', |
||
729 | 'Microsearch', |
||
730 | 'Microsoft\.Data\.Mashup', |
||
731 | 'Microsoft Office', |
||
732 | 'Microsoft Outlook', |
||
733 | 'Microsoft Windows Network Diagnostics', |
||
734 | 'Microsoft-WebDAV-MiniRedir', |
||
735 | 'Microsoft Data Access', |
||
736 | 'MIDown tool', |
||
737 | 'MIIxpc', |
||
738 | 'Mindjet', |
||
739 | 'Miniature\.io', |
||
740 | 'Miniflux', |
||
741 | 'Mister PiX', |
||
742 | 'mixdata dot com', |
||
743 | 'mixed-content-scan', |
||
744 | 'mixnode', |
||
745 | 'Mnogosearch', |
||
746 | 'mogimogi', |
||
747 | 'Mojeek', |
||
748 | 'Mojolicious \(Perl\)', |
||
749 | 'Monit\/', |
||
750 | 'monitis', |
||
751 | 'Monitority\/', |
||
752 | 'montastic', |
||
753 | 'MonTools', |
||
754 | 'Moreover', |
||
755 | 'Morfeus Fucking Scanner', |
||
756 | 'Morning Paper', |
||
757 | 'MovableType', |
||
758 | 'mowser', |
||
759 | 'Mr\.4x3 Powered', |
||
760 | 'Mrcgiguy', |
||
761 | 'MS Web Services Client Protocol', |
||
762 | 'MSFrontPage', |
||
763 | 'mShots', |
||
764 | 'MuckRack\/', |
||
765 | 'muhstik-scan', |
||
766 | 'MVAClient', |
||
767 | 'MxToolbox\/', |
||
768 | 'myseosnapshot', |
||
769 | 'nagios', |
||
770 | 'Najdi\.si', |
||
771 | 'Name Intelligence', |
||
772 | 'NameFo\.com', |
||
773 | 'Nameprotect', |
||
774 | 'nationalarchives', |
||
775 | 'Navroad', |
||
776 | 'NearSite', |
||
777 | 'Needle', |
||
778 | 'Nessus', |
||
779 | 'Net Vampire', |
||
780 | 'NetAnts', |
||
781 | 'NETCRAFT', |
||
782 | 'NetLyzer', |
||
783 | 'NetMechanic', |
||
784 | 'NetNewsWire', |
||
785 | 'Netpursual', |
||
786 | 'netresearch', |
||
787 | 'NetShelter ContentScan', |
||
788 | 'Netsparker', |
||
789 | 'NetTrack', |
||
790 | 'Netvibes', |
||
791 | 'NetZIP', |
||
792 | 'Neustar WPM', |
||
793 | 'NeutrinoAPI', |
||
794 | 'NewRelicPinger', |
||
795 | 'NewsBlur .*Finder', |
||
796 | 'NewsGator', |
||
797 | 'newsme', |
||
798 | 'newspaper\/', |
||
799 | 'NetSystemsResearch', |
||
800 | 'Nexgate Ruby Client', |
||
801 | 'nghttp2', |
||
802 | 'NG-Search', |
||
803 | 'Nibbler', |
||
804 | 'NICErsPRO', |
||
805 | 'Nikto', |
||
806 | 'nineconnections', |
||
807 | 'NLNZ_IAHarvester', |
||
808 | 'Nmap Scripting Engine', |
||
809 | 'node-superagent', |
||
810 | 'node-urllib', |
||
811 | 'node\.io', |
||
812 | 'Nodemeter', |
||
813 | 'NodePing', |
||
814 | 'nominet\.org\.uk', |
||
815 | 'nominet\.uk', |
||
816 | 'Norton-Safeweb', |
||
817 | 'Notifixious', |
||
818 | 'notifyninja', |
||
819 | 'NotionEmbedder', |
||
820 | 'nuhk', |
||
821 | 'nutch', |
||
822 | 'Nuzzel', |
||
823 | 'nWormFeedFinder', |
||
824 | 'nyawc\/', |
||
825 | 'Nymesis', |
||
826 | 'NYU', |
||
827 | 'Observatory\/', |
||
828 | 'Ocelli\/', |
||
829 | 'Octopus', |
||
830 | 'oegp', |
||
831 | 'Offline Explorer', |
||
832 | 'Offline Navigator', |
||
833 | 'OgScrper', |
||
834 | 'okhttp', |
||
835 | 'omgili', |
||
836 | 'OMSC', |
||
837 | 'Online Domain Tools', |
||
838 | 'OpenCalaisSemanticProxy', |
||
839 | 'Openfind', |
||
840 | 'OpenLinkProfiler', |
||
841 | 'Openstat\/', |
||
842 | 'OpenVAS', |
||
843 | 'OPPO A33', |
||
844 | 'Optimizer', |
||
845 | 'Open Source RSS', |
||
846 | 'Orbiter', |
||
847 | 'OrgProbe\/', |
||
848 | 'orion-semantics', |
||
849 | 'Outlook-Express', |
||
850 | 'Outlook-iOS', |
||
851 | 'ow\.ly', |
||
852 | 'Owler', |
||
853 | 'Owlin', |
||
854 | 'ownCloud News', |
||
855 | 'OxfordCloudService', |
||
856 | 'Page Valet', |
||
857 | 'page_verifier', |
||
858 | 'page scorer', |
||
859 | 'page2rss', |
||
860 | 'PageFreezer', |
||
861 | 'PageGrabber', |
||
862 | 'PagePeeker', |
||
863 | 'PageScorer', |
||
864 | 'PageThing', |
||
865 | 'Pagespeed\/', |
||
866 | 'Panopta', |
||
867 | 'panscient', |
||
868 | 'Papa Foto', |
||
869 | 'parsijoo', |
||
870 | 'Pavuk', |
||
871 | 'PayPal IPN', |
||
872 | 'pcBrowser', |
||
873 | 'Pcore-HTTP', |
||
874 | 'PDF24 URL To PDF', |
||
875 | 'Pearltrees', |
||
876 | 'PECL::HTTP', |
||
877 | 'peerindex', |
||
878 | 'Peew', |
||
879 | 'PeoplePal', |
||
880 | 'Perlu -', |
||
881 | 'PhantomJS Screenshoter', |
||
882 | 'PhantomJS\/', |
||
883 | 'Photon\/', |
||
884 | 'phpservermon', |
||
885 | 'php-requests', |
||
886 | 'Pi-Monster', |
||
887 | 'Picscout', |
||
888 | 'Picsearch', |
||
889 | 'PictureFinder', |
||
890 | 'Pimonster', |
||
891 | 'ping\.blo\.gs', |
||
892 | 'Pingability', |
||
893 | 'PingAdmin\.Ru', |
||
894 | 'Pingdom', |
||
895 | 'Pingoscope', |
||
896 | 'PingSpot', |
||
897 | 'pinterest\.com', |
||
898 | 'Pixray', |
||
899 | 'Pizilla', |
||
900 | 'Plagger\/', |
||
901 | 'Pleroma ', |
||
902 | 'Ploetz \+ Zeller', |
||
903 | 'Plukkie', |
||
904 | 'plumanalytics', |
||
905 | 'PocketImageCache', |
||
906 | 'PocketParser', |
||
907 | 'Pockey', |
||
908 | 'PodcastAddict\/', |
||
909 | 'POE-Component-Client-HTTP', |
||
910 | 'Polymail\/', |
||
911 | 'Pompos', |
||
912 | 'Porkbun', |
||
913 | 'Port Monitor', |
||
914 | 'postano', |
||
915 | 'postfix-mta-sts-resolver', |
||
916 | 'PostmanRuntime', |
||
917 | 'postplanner\.com', |
||
918 | 'PostPost', |
||
919 | 'postrank', |
||
920 | 'PowerPoint\/', |
||
921 | 'Prebid', |
||
922 | 'Prerender', |
||
923 | 'Priceonomics Analysis Engine', |
||
924 | 'PrintFriendly', |
||
925 | 'PritTorrent', |
||
926 | 'Prlog', |
||
927 | 'probethenet', |
||
928 | 'Project ?25499', |
||
929 | 'Project-Resonance', |
||
930 | 'prospectb2b', |
||
931 | 'Protopage', |
||
932 | 'ProWebWalker', |
||
933 | 'proximic', |
||
934 | 'PRTG Network Monitor', |
||
935 | 'pshtt, https scanning', |
||
936 | 'PTST ', |
||
937 | 'PTST\/[0-9]+', |
||
938 | 'Pump', |
||
939 | 'python-httpx', |
||
940 | 'Python-httplib2', |
||
941 | 'python-requests', |
||
942 | 'Python-urllib', |
||
943 | 'Qirina Hurdler', |
||
944 | 'QQDownload', |
||
945 | 'QrafterPro', |
||
946 | 'Qseero', |
||
947 | 'Qualidator', |
||
948 | 'QueryN Metasearch', |
||
949 | 'queuedriver', |
||
950 | 'quic-go-HTTP\/', |
||
951 | 'QuiteRSS', |
||
952 | 'Quora Link Preview', |
||
953 | 'Qwantify', |
||
954 | 'Radian6', |
||
955 | 'Railgun\/', |
||
956 | 'RankActive', |
||
957 | 'RankFlex', |
||
958 | 'RankSonicSiteAuditor', |
||
959 | 'Re-re Studio', |
||
960 | 'ReactorNetty', |
||
961 | 'Readability', |
||
962 | 'RealDownload', |
||
963 | 'RealPlayer%20Downloader', |
||
964 | 'RebelMouse', |
||
965 | 'Recorder', |
||
966 | 'RecurPost\/', |
||
967 | 'redback\/', |
||
968 | 'ReederForMac', |
||
969 | 'Reeder\/', |
||
970 | 'ReGet', |
||
971 | 'RepoMonkey', |
||
972 | 'request\.js', |
||
973 | 'reqwest\/', |
||
974 | 'ResponseCodeTest', |
||
975 | 'RestSharp', |
||
976 | 'Riddler', |
||
977 | 'Rival IQ', |
||
978 | 'Robosourcer', |
||
979 | 'Robozilla', |
||
980 | 'ROI Hunter', |
||
981 | 'RPT-HTTPClient', |
||
982 | 'RSSOwl', |
||
983 | 'RSSMix\/', |
||
984 | 'RyowlEngine', |
||
985 | 'safe-agent-scanner', |
||
986 | 'SalesIntelligent', |
||
987 | 'Saleslift', |
||
988 | 'Sendsay\.Ru', |
||
989 | 'SauceNAO', |
||
990 | 'SBIder', |
||
991 | 'sc-downloader', |
||
992 | 'scalaj-http', |
||
993 | 'Scamadviser-Frontend', |
||
994 | 'scan\.lol', |
||
995 | 'ScanAlert', |
||
996 | 'Scoop', |
||
997 | 'scooter', |
||
998 | 'ScoutJet', |
||
999 | 'ScopeContentAG-HTTP-Client', |
||
1000 | 'ScoutURLMonitor', |
||
1001 | 'ScrapeBox Page Scanner', |
||
1002 | 'Scrapy', |
||
1003 | 'Screaming', |
||
1004 | 'ScreenShotService', |
||
1005 | 'Scrubby', |
||
1006 | 'Scrutiny\/', |
||
1007 | 'search\.thunderstone', |
||
1008 | 'Search37', |
||
1009 | 'searchenginepromotionhelp', |
||
1010 | 'Searchestate', |
||
1011 | 'SearchExpress', |
||
1012 | 'SearchSight', |
||
1013 | 'SearchWP', |
||
1014 | 'Seeker', |
||
1015 | 'semanticdiscovery', |
||
1016 | 'semanticjuice', |
||
1017 | 'Semiocast HTTP client', |
||
1018 | 'Semrush', |
||
1019 | 'sentry\/', |
||
1020 | 'SEO Browser', |
||
1021 | 'Seo Servis', |
||
1022 | 'seo-nastroj\.cz', |
||
1023 | 'seo4ajax', |
||
1024 | 'Seobility', |
||
1025 | 'SEOCentro', |
||
1026 | 'SeoCheck', |
||
1027 | 'SEOkicks', |
||
1028 | 'SEOlizer', |
||
1029 | 'Seomoz', |
||
1030 | 'SEOprofiler', |
||
1031 | 'SEOsearch', |
||
1032 | 'seoscanners', |
||
1033 | 'seositecheckup', |
||
1034 | 'SEOstats', |
||
1035 | 'servernfo', |
||
1036 | 'sexsearcher', |
||
1037 | 'Seznam', |
||
1038 | 'Shelob', |
||
1039 | 'Shodan', |
||
1040 | 'Shoppimon', |
||
1041 | 'ShopWiki', |
||
1042 | 'shortURL lengthener', |
||
1043 | 'ShortLinkTranslate', |
||
1044 | 'shrinktheweb', |
||
1045 | 'Sideqik', |
||
1046 | 'Siege', |
||
1047 | 'SimplePie', |
||
1048 | 'SimplyFast', |
||
1049 | 'Siphon', |
||
1050 | 'SISTRIX', |
||
1051 | 'Site-Shot\/', |
||
1052 | 'Site Sucker', |
||
1053 | 'Site24x7', |
||
1054 | 'SiteBar', |
||
1055 | 'Sitebeam', |
||
1056 | 'Sitebulb\/', |
||
1057 | 'SiteCondor', |
||
1058 | 'SiteExplorer', |
||
1059 | 'SiteGuardian', |
||
1060 | 'Siteimprove', |
||
1061 | 'SiteIndexed', |
||
1062 | 'Sitemap(s)? Generator', |
||
1063 | 'SitemapGenerator', |
||
1064 | 'SiteMonitor', |
||
1065 | 'Siteshooter B0t', |
||
1066 | 'SiteSnagger', |
||
1067 | 'SiteSucker', |
||
1068 | 'SiteTruth', |
||
1069 | 'Sitevigil', |
||
1070 | 'sitexy\.com', |
||
1071 | 'SkypeUriPreview', |
||
1072 | 'Slack\/', |
||
1073 | 'slider\.com', |
||
1074 | 'sli-systems\.com', |
||
1075 | 'slurp', |
||
1076 | 'SlySearch', |
||
1077 | 'SmartDownload', |
||
1078 | 'SMRF URL Expander', |
||
1079 | 'SMUrlExpander', |
||
1080 | 'Snake', |
||
1081 | 'Snappy', |
||
1082 | 'SnapSearch', |
||
1083 | 'Snarfer\/', |
||
1084 | 'SniffRSS', |
||
1085 | 'sniptracker', |
||
1086 | 'Snoopy', |
||
1087 | 'SnowHaze Search', |
||
1088 | 'sogou web', |
||
1089 | 'SortSite', |
||
1090 | 'Sottopop', |
||
1091 | 'sovereign\.ai', |
||
1092 | 'SpaceBison', |
||
1093 | 'SpamExperts', |
||
1094 | 'Spammen', |
||
1095 | 'Spanner', |
||
1096 | 'spaziodati', |
||
1097 | 'SPDYCheck', |
||
1098 | 'Specificfeeds', |
||
1099 | 'speedy', |
||
1100 | 'SPEng', |
||
1101 | 'Spinn3r', |
||
1102 | 'spray-can', |
||
1103 | 'Sprinklr ', |
||
1104 | 'spyonweb', |
||
1105 | 'sqlmap', |
||
1106 | 'Sqlworm', |
||
1107 | 'Sqworm', |
||
1108 | 'SSL Labs', |
||
1109 | 'ssl-tools', |
||
1110 | 'StackRambler', |
||
1111 | 'Statastico\/', |
||
1112 | 'Statically-Screenshot', |
||
1113 | 'StatusCake', |
||
1114 | 'Steeler', |
||
1115 | 'Stratagems Kumo', |
||
1116 | 'Stripe\/', |
||
1117 | 'Stroke\.cz', |
||
1118 | 'StudioFACA', |
||
1119 | 'StumbleUpon', |
||
1120 | 'suchen', |
||
1121 | 'Sucuri', |
||
1122 | 'summify', |
||
1123 | 'SuperHTTP', |
||
1124 | 'Surphace Scout', |
||
1125 | 'Suzuran', |
||
1126 | 'swcd ', |
||
1127 | 'Symfony BrowserKit', |
||
1128 | 'Symfony2 BrowserKit', |
||
1129 | 'Syndirella\/', |
||
1130 | 'SynHttpClient-Built', |
||
1131 | 'Sysomos', |
||
1132 | 'sysscan', |
||
1133 | 'Szukacz', |
||
1134 | 'T0PHackTeam', |
||
1135 | 'tAkeOut', |
||
1136 | 'Tarantula\/', |
||
1137 | 'Taringa UGC', |
||
1138 | 'TarmotGezgin', |
||
1139 | 'tchelebi\.io', |
||
1140 | 'techiaith\.cymru', |
||
1141 | 'Teleport', |
||
1142 | 'Telesoft', |
||
1143 | 'Telesphoreo', |
||
1144 | 'Telesphorep', |
||
1145 | 'Tenon\.io', |
||
1146 | 'teoma', |
||
1147 | 'terrainformatica', |
||
1148 | 'Test Certificate Info', |
||
1149 | 'testuri', |
||
1150 | 'Tetrahedron', |
||
1151 | 'TextRazor Downloader', |
||
1152 | 'The Drop Reaper', |
||
1153 | 'The Expert HTML Source Viewer', |
||
1154 | 'The Knowledge AI', |
||
1155 | 'The Intraformant', |
||
1156 | 'theinternetrules', |
||
1157 | 'TheNomad', |
||
1158 | 'Thinklab', |
||
1159 | 'Thumbshots', |
||
1160 | 'ThumbSniper', |
||
1161 | 'Thumbor', |
||
1162 | 'timewe\.net', |
||
1163 | 'TinEye', |
||
1164 | 'Tiny Tiny RSS', |
||
1165 | 'TLSProbe\/', |
||
1166 | 'Toata', |
||
1167 | 'topster', |
||
1168 | 'touche\.com', |
||
1169 | 'Traackr\.com', |
||
1170 | 'tracemyfile', |
||
1171 | 'Trackuity', |
||
1172 | 'TrapitAgent', |
||
1173 | 'Trendiction', |
||
1174 | 'Trendsmap', |
||
1175 | 'trendspottr', |
||
1176 | 'truwoGPS', |
||
1177 | 'TryJsoup', |
||
1178 | 'TulipChain', |
||
1179 | 'Turingos', |
||
1180 | 'Turnitin', |
||
1181 | 'tweetedtimes', |
||
1182 | 'Tweetminster', |
||
1183 | 'Tweezler\/', |
||
1184 | 'twibble', |
||
1185 | 'Twice', |
||
1186 | 'Twikle', |
||
1187 | 'Twingly', |
||
1188 | 'Twisted PageGetter', |
||
1189 | 'Typhoeus', |
||
1190 | 'ubermetrics-technologies', |
||
1191 | 'uclassify', |
||
1192 | 'UdmSearch', |
||
1193 | 'unchaos', |
||
1194 | 'unirest-java', |
||
1195 | 'ultimate_sitemap_parser', |
||
1196 | 'UniversalFeedParser', |
||
1197 | 'Unshorten\.It', |
||
1198 | 'Untiny', |
||
1199 | 'UnwindFetchor', |
||
1200 | 'updated', |
||
1201 | 'updown\.io daemon', |
||
1202 | 'Upflow', |
||
1203 | 'Uptimia', |
||
1204 | 'Urlcheckr', |
||
1205 | 'URL Verifier', |
||
1206 | 'URLitor', |
||
1207 | 'urlresolver', |
||
1208 | 'Urlstat', |
||
1209 | 'URLTester', |
||
1210 | 'UrlTrends Ranking Updater', |
||
1211 | 'URLy Warning', |
||
1212 | 'URLy\.Warning', |
||
1213 | 'Vacuum', |
||
1214 | 'Vagabondo', |
||
1215 | 'VB Project', |
||
1216 | 'vBSEO', |
||
1217 | 'VCI', |
||
1218 | 'via ggpht\.com GoogleImageProxy', |
||
1219 | 'Virusdie', |
||
1220 | 'visionutils', |
||
1221 | 'vkShare', |
||
1222 | 'VoidEYE', |
||
1223 | 'Voil', |
||
1224 | 'voltron', |
||
1225 | 'voyager\/', |
||
1226 | 'VSAgent\/', |
||
1227 | 'VSB-TUO\/', |
||
1228 | 'Vulnbusters Meter', |
||
1229 | 'VYU2', |
||
1230 | 'w3af\.org', |
||
1231 | 'W3C_Unicorn', |
||
1232 | 'W3C-checklink', |
||
1233 | 'W3C-mobileOK', |
||
1234 | 'WAC-OFU', |
||
1235 | 'Wallpapers\/[0-9]+', |
||
1236 | 'WallpapersHD', |
||
1237 | 'WakeletLinkExpander', |
||
1238 | 'wangling', |
||
1239 | 'Wappalyzer', |
||
1240 | 'WatchMouse', |
||
1241 | 'WbSrch\/', |
||
1242 | 'WDT\.io', |
||
1243 | 'web-capture\.net', |
||
1244 | 'Web-sniffer', |
||
1245 | 'Web Auto', |
||
1246 | 'Web Collage', |
||
1247 | 'Web Enhancer', |
||
1248 | 'Web Fetch', |
||
1249 | 'Web Fuck', |
||
1250 | 'Web Pix', |
||
1251 | 'Web Sauger', |
||
1252 | 'Web spyder', |
||
1253 | 'Web Sucker', |
||
1254 | 'Webalta', |
||
1255 | 'Webauskunft', |
||
1256 | 'WebAuto', |
||
1257 | 'WebCapture', |
||
1258 | 'WebClient\/', |
||
1259 | 'webcollage', |
||
1260 | 'WebCookies', |
||
1261 | 'WebCopier', |
||
1262 | 'WebCorp', |
||
1263 | 'WebDataStats', |
||
1264 | 'WebDoc', |
||
1265 | 'WebEnhancer', |
||
1266 | 'WebFetch', |
||
1267 | 'WebFuck', |
||
1268 | 'WebGazer', |
||
1269 | 'WebGo IS', |
||
1270 | 'WebImageCollector', |
||
1271 | 'WebImages', |
||
1272 | 'WebIndex', |
||
1273 | 'webkit2png', |
||
1274 | 'WebLeacher', |
||
1275 | 'webmastercoffee', |
||
1276 | 'webmon ', |
||
1277 | 'WebPix', |
||
1278 | 'WebReaper', |
||
1279 | 'WebSauger', |
||
1280 | 'webscreenie', |
||
1281 | 'Webshag', |
||
1282 | 'Webshot', |
||
1283 | 'Website Quester', |
||
1284 | 'websitepulse agent', |
||
1285 | 'WebsiteQuester', |
||
1286 | 'Websnapr', |
||
1287 | 'WebSniffer', |
||
1288 | 'Webster', |
||
1289 | 'WebStripper', |
||
1290 | 'WebSucker', |
||
1291 | 'webtech\/', |
||
1292 | 'Webthumb\/', |
||
1293 | 'WebThumbnail', |
||
1294 | 'WebWhacker', |
||
1295 | 'WebZIP', |
||
1296 | 'WeLikeLinks', |
||
1297 | 'WEPA', |
||
1298 | 'WeSEE', |
||
1299 | 'wf84', |
||
1300 | 'Wfuzz\/', |
||
1301 | 'wget', |
||
1302 | 'WhatCMS', |
||
1303 | 'WhatsApp', |
||
1304 | 'WhatsMyIP', |
||
1305 | 'WhatWeb', |
||
1306 | 'WhereGoes\?', |
||
1307 | 'Whibse', |
||
1308 | 'WhoAPI\/', |
||
1309 | 'WhoRunsCoinHive', |
||
1310 | 'Whynder Magnet', |
||
1311 | 'WinHTTP\/', |
||
1312 | 'WinHttp-Autoproxy-Service', |
||
1313 | 'Windows-RSS-Platform', |
||
1314 | 'WinPodder', |
||
1315 | 'wkhtmlto', |
||
1316 | 'wmtips', |
||
1317 | 'Woko', |
||
1318 | 'Wolfram HTTPClient', |
||
1319 | 'woorankreview', |
||
1320 | 'Word\/', |
||
1321 | 'WordPress\/', |
||
1322 | 'worldping-api', |
||
1323 | 'WordupinfoSearch', |
||
1324 | 'wotbox', |
||
1325 | 'WP Engine Install Performance API', |
||
1326 | 'WP Rocket', |
||
1327 | 'wpif', |
||
1328 | 'wprecon\.com survey', |
||
1329 | 'WPScan', |
||
1330 | 'wscheck', |
||
1331 | 'Wtrace', |
||
1332 | 'WWW-Collector-E', |
||
1333 | 'WWW-Mechanize', |
||
1334 | 'WWW::Document', |
||
1335 | 'WWW::Mechanize', |
||
1336 | 'www\.monitor\.us', |
||
1337 | 'WWWOFFLE', |
||
1338 | 'x09Mozilla', |
||
1339 | 'x22Mozilla', |
||
1340 | 'XaxisSemanticsClassifier', |
||
1341 | 'XenForo\/', |
||
1342 | 'Xenu Link Sleuth', |
||
1343 | 'XING-contenttabreceiver', |
||
1344 | 'xpymep([0-9]?)\.exe', |
||
1345 | 'Y!J-(ASR|BSC)', |
||
1346 | 'Y\!J-BRW', |
||
1347 | 'Yaanb', |
||
1348 | 'yacy', |
||
1349 | 'Yahoo Link Preview', |
||
1350 | 'YahooCacheSystem', |
||
1351 | 'YahooMailProxy', |
||
1352 | 'YahooYSMcm', |
||
1353 | 'YandeG', |
||
1354 | 'Yandex(?!Search)', |
||
1355 | 'yanga', |
||
1356 | 'yeti', |
||
1357 | 'Yo-yo', |
||
1358 | 'Yoleo Consumer', |
||
1359 | 'yomins\.com', |
||
1360 | 'yoogliFetchAgent', |
||
1361 | 'YottaaMonitor', |
||
1362 | 'Your-Website-Sucks', |
||
1363 | 'yourls\.org', |
||
1364 | 'YoYs\.net', |
||
1365 | 'YP\.PL', |
||
1366 | 'Zabbix', |
||
1367 | 'Zade', |
||
1368 | 'Zao', |
||
1369 | 'Zauba', |
||
1370 | 'Zemanta Aggregator', |
||
1371 | 'Zend_Http_Client', |
||
1372 | 'Zend\\\\Http\\\\Client', |
||
1373 | 'Zermelo', |
||
1374 | 'Zeus ', |
||
1375 | 'zgrab', |
||
1376 | 'ZnajdzFoto', |
||
1377 | 'ZnHTTP', |
||
1378 | 'Zombie\.js', |
||
1379 | 'Zoom\.Mac', |
||
1380 | 'ZoteroTranslationServer', |
||
1381 | 'ZyBorg', |
||
1382 | '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)', |
||
1383 | ); |
||
1384 | } |
||
1385 |