diff --git a/modules/common/src/main/HTTPRequest.scala b/modules/common/src/main/HTTPRequest.scala index de37e2f06523..3276106f78d6 100644 --- a/modules/common/src/main/HTTPRequest.scala +++ b/modules/common/src/main/HTTPRequest.scala @@ -67,7 +67,7 @@ object HTTPRequest: private val crawlerMatcher = UaMatcher: // spiders/crawlers - """Googlebot|AdsBot|Google-Read-Aloud|bingbot|BingPreview|facebookexternalhit|SemrushBot|AhrefsBot|PetalBot|Applebot|YandexBot|YandexAdNet|Twitterbot|Baiduspider|Amazonbot|Bytespider|yacybot|ImagesiftBot|Ecosia""" + + """Googlebot|AdsBot|Google-Read-Aloud|bingbot|BingPreview|facebookexternalhit|SemrushBot|AhrefsBot|PetalBot|Applebot|YandexBot|YandexAdNet|Twitterbot|Baiduspider|Amazonbot|Bytespider|yacybot|ImagesiftBot|Ecosia|ChatGLM-Spider|YisouSpider""" + // http libs """|HeadlessChrome|okhttp|axios|wget|curl|python-requests|aiohttp|commons-httpclient|python-urllib|python-httpx|Nessus"""