| @@ -57,52 +57,52 @@ | ||
| 57 | 57 | * Instead, the list from version 1.26 is copied here. | 
| 58 | 58 | */ | 
| 59 | 59 | public const AI_ROBOTS = [ | 
| 60 | - 'AI2Bot', | |
| 61 | - 'Ai2Bot-Dolma', | |
| 62 | - 'Amazonbot', | |
| 63 | - 'anthropic-ai', | |
| 64 | - 'Applebot', | |
| 65 | - 'Applebot-Extended', | |
| 66 | - 'Brightbot 1.0', | |
| 67 | - 'Bytespider', | |
| 68 | - 'CCBot', | |
| 69 | - 'ChatGPT-User', | |
| 70 | - 'Claude-Web', | |
| 71 | - 'ClaudeBot', | |
| 72 | - 'cohere-ai', | |
| 73 | - 'cohere-training-data-crawler', | |
| 74 | - 'Crawlspace', | |
| 75 | - 'Diffbot', | |
| 76 | - 'DuckAssistBot', | |
| 77 | - 'FacebookBot', | |
| 78 | - 'FriendlyCrawler', | |
| 79 | - 'Google-Extended', | |
| 80 | - 'GoogleOther', | |
| 81 | - 'GoogleOther-Image', | |
| 82 | - 'GoogleOther-Video', | |
| 83 | - 'GPTBot', | |
| 84 | - 'iaskspider/2.0', | |
| 85 | - 'ICC-Crawler', | |
| 86 | - 'ImagesiftBot', | |
| 87 | - 'img2dataset', | |
| 88 | - 'ISSCyberRiskCrawler', | |
| 89 | - 'Kangaroo Bot', | |
| 90 | - 'Meta-ExternalAgent', | |
| 91 | - 'Meta-ExternalFetcher', | |
| 92 | - 'OAI-SearchBot', | |
| 93 | - 'omgili', | |
| 94 | - 'omgilibot', | |
| 95 | - 'PanguBot', | |
| 96 | - 'PerplexityBot', | |
| 97 | - 'PetalBot', | |
| 98 | - 'Scrapy', | |
| 99 | - 'SemrushBot-OCOB', | |
| 100 | - 'SemrushBot-SWA', | |
| 101 | - 'Sidetrade indexer bot', | |
| 102 | - 'Timpibot', | |
| 103 | - 'VelenPublicWebCrawler', | |
| 104 | - 'Webzio-Extended', | |
| 105 | - 'YouBot', | |
| 60 | + 'AI2Bot', | |
| 61 | + 'Ai2Bot-Dolma', | |
| 62 | + 'Amazonbot', | |
| 63 | + 'anthropic-ai', | |
| 64 | + 'Applebot', | |
| 65 | + 'Applebot-Extended', | |
| 66 | + 'Brightbot 1.0', | |
| 67 | + 'Bytespider', | |
| 68 | + 'CCBot', | |
| 69 | + 'ChatGPT-User', | |
| 70 | + 'Claude-Web', | |
| 71 | + 'ClaudeBot', | |
| 72 | + 'cohere-ai', | |
| 73 | + 'cohere-training-data-crawler', | |
| 74 | + 'Crawlspace', | |
| 75 | + 'Diffbot', | |
| 76 | + 'DuckAssistBot', | |
| 77 | + 'FacebookBot', | |
| 78 | + 'FriendlyCrawler', | |
| 79 | + 'Google-Extended', | |
| 80 | + 'GoogleOther', | |
| 81 | + 'GoogleOther-Image', | |
| 82 | + 'GoogleOther-Video', | |
| 83 | + 'GPTBot', | |
| 84 | + 'iaskspider/2.0', | |
| 85 | + 'ICC-Crawler', | |
| 86 | + 'ImagesiftBot', | |
| 87 | + 'img2dataset', | |
| 88 | + 'ISSCyberRiskCrawler', | |
| 89 | + 'Kangaroo Bot', | |
| 90 | + 'Meta-ExternalAgent', | |
| 91 | + 'Meta-ExternalFetcher', | |
| 92 | + 'OAI-SearchBot', | |
| 93 | + 'omgili', | |
| 94 | + 'omgilibot', | |
| 95 | + 'PanguBot', | |
| 96 | + 'PerplexityBot', | |
| 97 | + 'PetalBot', | |
| 98 | + 'Scrapy', | |
| 99 | + 'SemrushBot-OCOB', | |
| 100 | + 'SemrushBot-SWA', | |
| 101 | + 'Sidetrade indexer bot', | |
| 102 | + 'Timpibot', | |
| 103 | + 'VelenPublicWebCrawler', | |
| 104 | + 'Webzio-Extended', | |
| 105 | + 'YouBot', | |
| 106 | 106 | ]; | 
| 107 | 107 | |
| 108 | 108 | // Other bad robots - SEO optimisers, advertisers, etc. This list is shared with robots.txt. |