@@ -57,52 +57,52 @@ |
||
57 | 57 | * Instead, the list from version 1.26 is copied here. |
58 | 58 | */ |
59 | 59 | public const array AI_ROBOTS = [ |
60 | - 'AI2Bot', |
|
61 | - 'Ai2Bot-Dolma', |
|
62 | - 'Amazonbot', |
|
63 | - 'anthropic-ai', |
|
64 | - 'Applebot', |
|
65 | - 'Applebot-Extended', |
|
66 | - 'Brightbot 1.0', |
|
67 | - 'Bytespider', |
|
68 | - 'CCBot', |
|
69 | - 'ChatGPT-User', |
|
70 | - 'Claude-Web', |
|
71 | - 'ClaudeBot', |
|
72 | - 'cohere-ai', |
|
73 | - 'cohere-training-data-crawler', |
|
74 | - 'Crawlspace', |
|
75 | - 'Diffbot', |
|
76 | - 'DuckAssistBot', |
|
77 | - 'FacebookBot', |
|
78 | - 'FriendlyCrawler', |
|
79 | - 'Google-Extended', |
|
80 | - 'GoogleOther', |
|
81 | - 'GoogleOther-Image', |
|
82 | - 'GoogleOther-Video', |
|
83 | - 'GPTBot', |
|
84 | - 'iaskspider/2.0', |
|
85 | - 'ICC-Crawler', |
|
86 | - 'ImagesiftBot', |
|
87 | - 'img2dataset', |
|
88 | - 'ISSCyberRiskCrawler', |
|
89 | - 'Kangaroo Bot', |
|
90 | - 'meta-externalagent', |
|
91 | - 'meta-externalfetcher', |
|
92 | - 'OAI-SearchBot', |
|
93 | - 'omgili', |
|
94 | - 'omgilibot', |
|
95 | - 'PanguBot', |
|
96 | - 'PerplexityBot', |
|
97 | - 'PetalBot', |
|
98 | - 'Scrapy', |
|
99 | - 'SemrushBot-OCOB', |
|
100 | - 'SemrushBot-SWA', |
|
101 | - 'Sidetrade indexer bot', |
|
102 | - 'Timpibot', |
|
103 | - 'VelenPublicWebCrawler', |
|
104 | - 'Webzio-Extended', |
|
105 | - 'YouBot', |
|
60 | + 'AI2Bot', |
|
61 | + 'Ai2Bot-Dolma', |
|
62 | + 'Amazonbot', |
|
63 | + 'anthropic-ai', |
|
64 | + 'Applebot', |
|
65 | + 'Applebot-Extended', |
|
66 | + 'Brightbot 1.0', |
|
67 | + 'Bytespider', |
|
68 | + 'CCBot', |
|
69 | + 'ChatGPT-User', |
|
70 | + 'Claude-Web', |
|
71 | + 'ClaudeBot', |
|
72 | + 'cohere-ai', |
|
73 | + 'cohere-training-data-crawler', |
|
74 | + 'Crawlspace', |
|
75 | + 'Diffbot', |
|
76 | + 'DuckAssistBot', |
|
77 | + 'FacebookBot', |
|
78 | + 'FriendlyCrawler', |
|
79 | + 'Google-Extended', |
|
80 | + 'GoogleOther', |
|
81 | + 'GoogleOther-Image', |
|
82 | + 'GoogleOther-Video', |
|
83 | + 'GPTBot', |
|
84 | + 'iaskspider/2.0', |
|
85 | + 'ICC-Crawler', |
|
86 | + 'ImagesiftBot', |
|
87 | + 'img2dataset', |
|
88 | + 'ISSCyberRiskCrawler', |
|
89 | + 'Kangaroo Bot', |
|
90 | + 'meta-externalagent', |
|
91 | + 'meta-externalfetcher', |
|
92 | + 'OAI-SearchBot', |
|
93 | + 'omgili', |
|
94 | + 'omgilibot', |
|
95 | + 'PanguBot', |
|
96 | + 'PerplexityBot', |
|
97 | + 'PetalBot', |
|
98 | + 'Scrapy', |
|
99 | + 'SemrushBot-OCOB', |
|
100 | + 'SemrushBot-SWA', |
|
101 | + 'Sidetrade indexer bot', |
|
102 | + 'Timpibot', |
|
103 | + 'VelenPublicWebCrawler', |
|
104 | + 'Webzio-Extended', |
|
105 | + 'YouBot', |
|
106 | 106 | ]; |
107 | 107 | |
108 | 108 | // Other bad robots - SEO optimisers, advertisers, etc. This list is shared with robots.txt. |