hexydec\agentzero\crawlers

Complexity

Total Complexity

11

Size/Duplication

Total Lines	387
Duplicated Lines	0 %

Test Coverage

Coverage

100%

Importance

Changes	4
Bugs	0	Features	0

2 Methods

Rating	Name	Duplication	Size	Complexity
B	get()	0	179	2
C	getApp()	0	192	9

<?php
declare(strict_types = 1);
namespace hexydec\agentzero;

class crawlers {

	/**
	 * Extracts application and version information from a token
	 * 
	 * @param string $value The token to be processed
	 * @param array<string|null> $data An array containing existing data to merge
	 * @return array<string|int|float|null> The $data array with the processed application and version added
	 */
	public static function getApp(string $value, array $data = []) : array {
		if (!\str_contains($value, '://') && !\str_starts_with($value, 'Chrome/') && \strcasecmp('Cubot', $value) !== 0 && \strcasecmp('Power bot', $value) !== 0) { // bot will be in the URL
			$parts = \explode('/', $value, 2);

			// process version
			if (!empty($parts[1])) {
				$parts[1] = \ltrim($parts[1], 'v');
				$parts[1] = \substr($parts[1], 0, \strspn($parts[1], '0123456789.'));
			}
			$category = [
				'yacybot' => [
					'category' => 'search',
					'app' => 'YacyBot'
				],
				'googlebot' => [
					'category' => 'search',
					'app' => 'GoogleBot'
				],
				'googlebot-mobile' => [
					'category' => 'search',
					'app' => 'GoogleBot'
				],
				'googlebot-image' => [
					'category' => 'search',
					'app' => 'GoogleBot'
				],
				'googlebot-video' => [
					'category' => 'search',
					'app' => 'GoogleBot'
				],
				'googlebot-news' => [
					'category' => 'search',
					'app' => 'GoogleBot'
				],
				'storebot-google' => [
					'category' => 'search',
					'app' => 'GoogleBot'
				],
				'adsbot-google' => [
					'category' => 'ads',
					'app' => 'GoogleBot'
				],
				'adsbot-google-mobile' => [
					'category' => 'ads',
					'app' => 'GoogleBot'
				],
				'mediapartners-google' => [
					'category' => 'ads',
					'app' => 'GoogleBot'
				],
				'bingbot' => [
					'category' => 'search',
					'app' => 'BingBot'
				],
				'adidxbot' => [
					'category' => 'ads',
					'app' => 'AdidxBot'
				],
				'duckduckbot' => [
					'category' => 'search',
					'app' => 'DuckDuckBot'
				],
				'duckduckgo-favicons-bot' => [
					'category' => 'search',
					'app' => 'DuckDuckBot'
				],
				'coccocbot-image' => [
					'category' => 'search',
					'app' => 'CoccocBot'
				],
				'coccocbot-web' => [
					'category' => 'search',
					'app' => 'CoccocBot'
				],
				'applebot' => [
					'category' => 'search',
					'app' => 'AppleBot'
				],
				'yandexbot' => [
					'category' => 'search'
				],
				'mj12bot' => [
					'category' => 'search',
					'app' => 'Majestic 12 Bot'
				],
				'mail.ru_bot' => [
					'category' => 'search',
					'app' => 'Mail.ru Bot'
				],
				'exabot' => [
					'category' => 'search',
					'app' => 'ExaBot'
				],
				'uptimerobot' => [
					'category' => 'monitor'
				],
				'petalbot' => [
					'category' => 'search'
				],
				'twitterbot' => [
					'category' => 'feed',
					'app' => 'TwitterBot'
				],
				'xbot' => [
					'category' => 'feed'
				],
				'discordbot' => [
					'category' => 'feed',
					'app' => 'DiscordBot'
				],
				'sematextsyntheticsrobot' => [
					'category' => 'monitor',
					'app' => 'Sematext Synthetics Robot'
				],
				'linkedinbot' => [
					'category' => 'feed'
				],
				'paperlibot' => [
					'category' => 'feed'
				],
				'bitlybot' => [
					'category' => 'feed',
					'app' => 'Bit.ly Bot'
				],
				'tineye-bot' => [
					'category' => 'search',
					'app' => 'TinEye Bot'
				],
				'pinterestbot' => [
					'category' => 'feed',
					'app' => 'PinterestBot'
				],
				'webcrawler' => [
					'category' => 'crawler'
				],
				'webprosbot' => [
					'category' => 'crawler',
					'app' => 'WebprosBot'
				],
				'guzzlehttp' => [
					'category' => 'scraper'
				],
				'telegrambot' => [
					'category' => 'feed'
				],
				'semrushbot' => [
					'category' => 'crawler'
				],
				'mediatoolkitbot' => [
					'category' => 'crawler',
					'app' => 'MediaToolkitBot'
				],
				'iploggerbot' => [
					'category' => 'monitor'
				],
				'cfnetwork' => [
					'category' => 'feed',
					'app' => 'Apple Core Foundation Network'
				],
				'ncsc web check feedback.webcheck@digital.ncsc.gov.uk' => [
					'category' => 'monitor',
					'app' => 'NCSC Web Check'
				],
				'google-site-verification' => [
					'category' => 'validator',
					'app' => 'Google Site Verification'
				],
				'google-inspectiontool' => [
					'category' => 'validator',
					'app' => 'Google Inspection Tool'
				],
				'pingdomtms' => [
					'category' => 'monitor',
					'app' => 'Pingdom.com'
				],
				'facebookexternalhit' => [
					'category' => 'feed',
					'app' => 'Facebook URL Preview'
				],
				'phxbot' => [
					'app' => 'ProtonMail Bot'
				]
			];
			return \array_merge([
				'type' => 'robot',
				'category' => \mb_stripos($value, 'crawler') !== false || \mb_stripos($value, 'bot') !== false ? 'crawler' : 'scraper',
				'app' => $parts[0],
				'appname' => $parts[0],
				'appversion' => empty($parts[1]) ? null : $parts[1]
			], $data, $category[\mb_strtolower($parts[0])] ?? []);
		}
		return [];
	}

	/**
	 * Generates a configuration array for matching crawlers
	 * 
	 * @return array<string,props> An array with keys representing the string to match, and values a props object defining how to generate the match and which properties to set
	 */
	public static function get() : array {
		$fn = [
			'search' => fn (string $value) : array => self::getApp($value, ['category' => 'search']),
			'ads' => fn (string $value) : array => self::getApp($value, ['category' => 'ads']),
			'validator' => fn (string $value) : array => self::getApp($value, ['category' => 'validator']),
			'feed' => fn (string $value) : array => self::getApp($value, \array_merge(
				\str_contains($value, 'WhatsApp/') ? [
					'app' => 'WhatsApp'
				] : [],
				[
					'category' => 'feed'
				]
			)),
			'crawler' => function (string $value) : array {
				$parts = \explode('/', $value, 2);
				$map = [
					'baiduspider' => 'search',
					'haosouspider' => 'search',
					'yisouspider' => 'search',
					'360spider' => 'search',
					'sogou web spider' => 'search',
					'bytespider' => 'search',
				];
				return self::getApp($value, [
					'category' => $map[\mb_strtolower($parts[0])] ?? 'crawler'
				]);
			},
			'monitor' => fn (string $value) : array => self::getApp($value, ['category' => 'monitor']),
			'scraper' => fn (string $value) : array => self::getApp($value, ['category' => 'scraper']),
			'map' => fn (string $value) : ?array => self::getApp($value)
		];
		return [
			'Yahoo! Slurp' => new props('exact', $fn['search']),
			'facebookexternalhit/' => new props('start', $fn['map']),
			'Google-Site-Verification/' => new props('start', $fn['map']),
			'Google-InspectionTool/' => new props('start', $fn['map']),
			'Mediapartners-Google' => new props('start', $fn['search']),
			'FeedFetcher-Google' => new props('exact', $fn['feed']),
			'GoogleProducer' => new props('exact', $fn['feed']),
			'Google-adstxt' => new props('exact', $fn['ads']),
			'CFNetwork/' => new props('start', $fn['map']),
			'Siteimprove.com' => new props('any', $fn['crawler']),
			'CyotekWebCopy' => new props('start', $fn['scraper']),
			'Google Page Speed Insights' => new props('exact', $fn['validator']),
			'Qwantify' => new props('start', function (string $value) : array {
				$parts = \explode('/', $value, 3);
				return [
					'type' => 'robot',
					'category' => 'search',
					'app' => 'Qwant Web Crawler',
					'appname' => $parts[0],
					'appversion' => $parts[1] ?? null
				];
			}),
			'okhttp' => new props('start', $fn['scraper']),
			'python' => new props('start', $fn['scraper']),
			'jsdom/' => new props('start', $fn['scraper']),
			'Nessus' => new props('start', $fn['monitor']),
			'Chrome-Lighthouse' => new props('start', $fn['validator']),
			'Siege/' => new props('start', $fn['validator']),
			'Microsoft Profiling/' => new props('any', $fn['validator']),
			'Bidtellect' => new props('start', $fn['crawler']),
			'magpie-crawler/' => new props('start', $fn['crawler']),
			'PingdomTMS/' => new props('start', $fn['map']),
			'DynGate' => new props('exact', $fn['monitor']),
			'Datadog/Synthetics' => new props('exact', [
				'type' => 'robot',
				'category' => 'monitor',
				'app' => 'Datadog/Synthetics'
			]),
			'RuxitSynthetic/' => new props('start', $fn['monitor']),
			'Checkly/' => new props('start', $fn['monitor']),
			'Uptime/' => new props('start', $fn['monitor']),
			'HostTracker/' => new props('start', $fn['monitor']),
			'NCSC Web Check feedback.webcheck@digital.ncsc.gov.uk' => new props('exact', $fn['map']),
			'Pingdom.com' => new props('start', function (string $value) : array {
				$version = \explode('_', \trim($value, '_'));
				return [
					'type' => 'robot',
					'category' => 'monitor',
					'app' => 'Pingdom.com',
					'appname' => \trim($value, '_'),
					'appversion' => \end($version)
				];
			}),
			'proximic' => new props('exact', $fn['ads']),
			'WordPress' => new props('start', $fn['monitor']),
			'PRTG Network Monitor' => new props('exact', $fn['monitor']),
			'PRTGCloudBot/' => new props('start', $fn['monitor']),
			'Site24x7' => new props('exact', $fn['monitor']),
			'StatusCake' => new props('exact', $fn['monitor']),
			'adbeat.com' => new props('start', fn (string $value) : array => [
				'type' => 'robot',
				'category' => 'ads',
				'app' => 'Adbeat',
				'appname' => 'Adbeat',
				'url' => 'https://'.$value
			]),
			'MicrosoftPreview/' => new props('start', $fn['feed']),
			'YahooMailProxy' => new props('exact', $fn['feed']),
			'PhxBot/' => new props('start', $fn['feed']), // proton mail
			'Pleroma' => new props('start', fn (string $value) : array => [ // mastodon
				'type' => 'robot',
				'category' => 'feed',
				'app' => 'Mastodon',
				'appname' => 'Pleroma',
				'appversion' => \mb_substr($value, 8)
			]),
			'Outlook-Android/' => new props('start', fn (string $value) : array => [ // mastodon
				'type' => 'robot',
				'category' => 'feed',
				'app' => 'Outlook',
				'appname' => 'Outlook-Android',
				'platform' => 'Android',
				'appversion' => \mb_substr($value, 16)
			]),
			'Outlook-iOS/' => new props('start', fn (string $value, int $i, array $tokens) : array => [ // mastodon
				'type' => 'robot',
				'category' => 'feed',
				'app' => 'Outlook',
				'appname' => 'Outlook-iOS',
				'platform' => 'iOS',
				'appversion' => $tokens[$i+1] ?? \mb_substr($value, 12)
			]),
			'OutlookMobileCloudService-Autodetect/' => new props('start', fn (string $value) : array => [ // mastodon
				'type' => 'robot',
				'category' => 'feed',
				'app' => 'Outlook',
				'appname' => 'OutlookMobileCloudService-Autodetect',
				'appversion' => \mb_substr($value, 37)
			]),
			'Chrome Privacy Preserving Prefetch Proxy' => new props('exact', $fn['feed']),
			'ViberUrlDownloader' => new props('exact', $fn['feed']),
			'Google-Lens' => new props('exact', $fn['feed']),
			'ManicTime/' => new props('start', $fn['feed']),
			'Yik Yak/' => new props('start', $fn['feed']),
			'HubSpot-Link-Resolver' => new props('exact', $fn['feed']),
			'W3C-checklink/' => new props('start', $fn['validator']),
			'CSSCheck/' => new props('start', $fn['validator']),
			'Let\'s Encrypt validation server' => new props('exact', $fn['validator']),
			'SEO-Macroscope/' => new props('exact', $fn['validator']),
			'Electronic Frontier Foundation\'s Do Not Track Verifier' => new props('exact', $fn['validator']),
			'Expanse' => new props('start', $fn['crawler']),
			'eCairn-Grabber/' => new props('start', $fn['scraper']),
			'SEOkicks' => new props('exact', $fn['crawler']),
			'PostmanRuntime/' => new props('start', $fn['scraper']),
			'axios/' => new props('start', $fn['scraper']),
			'Rogerbot/' => new props('start', $fn['crawler']),
			'DashLinkPreviews/' => new props('start', $fn['feed']),
			'Snapchat/' => new props('start', $fn['feed']),
			'HTTPClient/' => new props('start', $fn['scraper']),
			'WhatsApp/' => new props('any', $fn['feed']),
			'Hootsuite-Authoring/' => new props('start', $fn['feed']),
			'Snap URL Preview Service' => new props('exact', $fn['feed']),
			'ApacheBench/' => new props('start', $fn['validator']),
			'Asana/' => new props('start', $fn['feed']),
			'Java/' => new props('start', $fn['scraper']),
			'curl/' => new props('any', $fn['scraper']),
			'Wget/' => new props('start', $fn['scraper']),
			'rest-client/' => new props('start', $fn['scraper']),
			'ruby/' => new props('start', $fn['scraper']),
			'Bun/' => new props('start', $fn['scraper']),
			'CakePHP' => new props('start', $fn['scraper']),
			'cpp-httplib/' => new props('start', $fn['scraper']),
			'Dart/' => new props('start', $fn['scraper']),
			'Deno/' => new props('start', $fn['scraper']),
			'libwww-perl/' => new props('start', $fn['scraper']),
			'http/' => new props('start', $fn['scraper']),
			'Cpanel-HTTP-Client/' => new props('start', $fn['scraper']),
			'http-client/' => new props('any', $fn['scraper']),
			'HttpClient/' => new props('any', $fn['scraper']),
			'Validator' => new props('any', $fn['validator']),
			'feed' => new props('any', $fn['feed']),
			'spider' => new props('any', $fn['crawler']),
			'crawler' => new props('any', $fn['map']),
			'bot/' => new props('any', $fn['map']),
			'bot-' => new props('any', $fn['map']),
			' bot ' => new props('any', $fn['map']),
			'bot' => new props('end', $fn['map'])
		];
	}
}