@@ -57,52 +57,52 @@ |
||
| 57 | 57 | * Instead, the list from version 1.26 is copied here. |
| 58 | 58 | */ |
| 59 | 59 | public const AI_ROBOTS = [ |
| 60 | - 'AI2Bot', |
|
| 61 | - 'Ai2Bot-Dolma', |
|
| 62 | - 'Amazonbot', |
|
| 63 | - 'anthropic-ai', |
|
| 64 | - 'Applebot', |
|
| 65 | - 'Applebot-Extended', |
|
| 66 | - 'Brightbot 1.0', |
|
| 67 | - 'Bytespider', |
|
| 68 | - 'CCBot', |
|
| 69 | - 'ChatGPT-User', |
|
| 70 | - 'Claude-Web', |
|
| 71 | - 'ClaudeBot', |
|
| 72 | - 'cohere-ai', |
|
| 73 | - 'cohere-training-data-crawler', |
|
| 74 | - 'Crawlspace', |
|
| 75 | - 'Diffbot', |
|
| 76 | - 'DuckAssistBot', |
|
| 77 | - 'FacebookBot', |
|
| 78 | - 'FriendlyCrawler', |
|
| 79 | - 'Google-Extended', |
|
| 80 | - 'GoogleOther', |
|
| 81 | - 'GoogleOther-Image', |
|
| 82 | - 'GoogleOther-Video', |
|
| 83 | - 'GPTBot', |
|
| 84 | - 'iaskspider/2.0', |
|
| 85 | - 'ICC-Crawler', |
|
| 86 | - 'ImagesiftBot', |
|
| 87 | - 'img2dataset', |
|
| 88 | - 'ISSCyberRiskCrawler', |
|
| 89 | - 'Kangaroo Bot', |
|
| 90 | - 'meta-externalagent', |
|
| 91 | - 'meta-externalfetcher', |
|
| 92 | - 'OAI-SearchBot', |
|
| 93 | - 'omgili', |
|
| 94 | - 'omgilibot', |
|
| 95 | - 'PanguBot', |
|
| 96 | - 'PerplexityBot', |
|
| 97 | - 'PetalBot', |
|
| 98 | - 'Scrapy', |
|
| 99 | - 'SemrushBot-OCOB', |
|
| 100 | - 'SemrushBot-SWA', |
|
| 101 | - 'Sidetrade indexer bot', |
|
| 102 | - 'Timpibot', |
|
| 103 | - 'VelenPublicWebCrawler', |
|
| 104 | - 'Webzio-Extended', |
|
| 105 | - 'YouBot', |
|
| 60 | + 'AI2Bot', |
|
| 61 | + 'Ai2Bot-Dolma', |
|
| 62 | + 'Amazonbot', |
|
| 63 | + 'anthropic-ai', |
|
| 64 | + 'Applebot', |
|
| 65 | + 'Applebot-Extended', |
|
| 66 | + 'Brightbot 1.0', |
|
| 67 | + 'Bytespider', |
|
| 68 | + 'CCBot', |
|
| 69 | + 'ChatGPT-User', |
|
| 70 | + 'Claude-Web', |
|
| 71 | + 'ClaudeBot', |
|
| 72 | + 'cohere-ai', |
|
| 73 | + 'cohere-training-data-crawler', |
|
| 74 | + 'Crawlspace', |
|
| 75 | + 'Diffbot', |
|
| 76 | + 'DuckAssistBot', |
|
| 77 | + 'FacebookBot', |
|
| 78 | + 'FriendlyCrawler', |
|
| 79 | + 'Google-Extended', |
|
| 80 | + 'GoogleOther', |
|
| 81 | + 'GoogleOther-Image', |
|
| 82 | + 'GoogleOther-Video', |
|
| 83 | + 'GPTBot', |
|
| 84 | + 'iaskspider/2.0', |
|
| 85 | + 'ICC-Crawler', |
|
| 86 | + 'ImagesiftBot', |
|
| 87 | + 'img2dataset', |
|
| 88 | + 'ISSCyberRiskCrawler', |
|
| 89 | + 'Kangaroo Bot', |
|
| 90 | + 'meta-externalagent', |
|
| 91 | + 'meta-externalfetcher', |
|
| 92 | + 'OAI-SearchBot', |
|
| 93 | + 'omgili', |
|
| 94 | + 'omgilibot', |
|
| 95 | + 'PanguBot', |
|
| 96 | + 'PerplexityBot', |
|
| 97 | + 'PetalBot', |
|
| 98 | + 'Scrapy', |
|
| 99 | + 'SemrushBot-OCOB', |
|
| 100 | + 'SemrushBot-SWA', |
|
| 101 | + 'Sidetrade indexer bot', |
|
| 102 | + 'Timpibot', |
|
| 103 | + 'VelenPublicWebCrawler', |
|
| 104 | + 'Webzio-Extended', |
|
| 105 | + 'YouBot', |
|
| 106 | 106 | ]; |
| 107 | 107 | |
| 108 | 108 | // Other bad robots - SEO optimisers, advertisers, etc. This list is shared with robots.txt. |