diff --git a/validate.js b/validate.js index a5b8b0b..e13ad40 100644 --- a/validate.js +++ b/validate.js @@ -53,6 +53,10 @@ if (process.argv[2] === "--check") { console.error("Item has wrong type specified for `url` string field:", item); process.exit(1); } + if (!Array.isArray(item.verification)) { + console.error("Item is missing required `validation` array field:", item); + process.exit(1); + } // TODO: Check `addition_date` is defined properly // TODO: Check or remove `depends_on` field if (typeof item.instances !== "undefined") { diff --git a/well-known-bots.json b/well-known-bots.json index 0932044..400f71c 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -7,6 +7,15 @@ ], "pattern": "Googlebot\\/", "url": "http://www.google.com/bot.html", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "Googlebot/2.1 (+http://www.google.com/bot.html)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", @@ -25,6 +34,7 @@ "search-engine" ], "pattern": "Googlebot-Mobile", + "verification": [], "instances": [ "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", @@ -40,6 +50,15 @@ "search-engine" ], "pattern": "Googlebot-Image", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "Googlebot-Image/1.0" ] @@ -51,6 +70,15 @@ "search-engine" ], "pattern": "Googlebot-News", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "Googlebot-News" ] @@ -62,6 +90,15 @@ "search-engine" ], "pattern": "Googlebot-Video", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "Googlebot-Video/1.0" ] @@ -74,6 +111,14 @@ ], "pattern": "AdsBot-Google([^-]|$)", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "verification": [ + { + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" + ] + } + ], "instances": [ "AdsBot-Google (+http://www.google.com/adsbot.html)" ] @@ -87,6 +132,14 @@ "pattern": "AdsBot-Google-Mobile", "addition_date": "2017/08/21", "url": "https://support.google.com/adwords/answer/2404197", + "verification": [ + { + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" + ] + } + ], "instances": [ "AdsBot-Google-Mobile-Apps", "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", @@ -102,6 +155,15 @@ "pattern": "Feedfetcher-Google", "addition_date": "2018/06/27", "url": "https://support.google.com/webmasters/answer/178852", + "verification": [ + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], "instances": [ "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)" ] @@ -114,6 +176,14 @@ ], "pattern": "Mediapartners-Google", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "verification": [ + { + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" + ] + } + ], "instances": [ "Mediapartners-Google", "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", @@ -130,6 +200,7 @@ "pattern": "Mediapartners \\(Googlebot\\)", "addition_date": "2017/08/08", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "verification": [], "instances": [] }, { @@ -140,6 +211,14 @@ "pattern": "APIs-Google", "addition_date": "2017/08/08", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "verification": [ + { + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" + ] + } + ], "instances": [ "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)" ] @@ -152,6 +231,15 @@ ], "pattern": "Google-InspectionTool", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)", "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)" @@ -165,6 +253,15 @@ ], "pattern": "Storebot-Google", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36" @@ -178,6 +275,15 @@ ], "pattern": "GoogleOther", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "GoogleOther" ] @@ -190,6 +296,14 @@ ], "pattern": "bingbot", "url": "http://www.bing.com/bingbot.htm", + "verification": [ + { + "type": "dns", + "masks": [ + "@.search.msn.com" + ] + } + ], "instances": [ "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", @@ -215,6 +329,7 @@ ], "pattern": "Slurp", "url": "http://help.yahoo.com/help/us/ysearch/slurp", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", @@ -227,6 +342,7 @@ "tool" ], "pattern": "[wW]get", + "verification": [], "instances": [ "WGETbot/1.0 (+http://wget.alanreed.org)", "Wget/1.14 (linux-gnu)", @@ -239,6 +355,7 @@ "social" ], "pattern": "LinkedInBot", + "verification": [], "instances": [ "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)", @@ -251,6 +368,7 @@ "programmatic" ], "pattern": "Python-urllib", + "verification": [], "instances": [ "Python-urllib/1.17", "Python-urllib/2.5", @@ -272,6 +390,7 @@ ], "pattern": "python-requests", "addition_date": "2018/05/27", + "verification": [], "instances": [ "python-requests/2.9.2", "python-requests/2.11.1", @@ -289,6 +408,7 @@ ], "pattern": "aiohttp", "addition_date": "2019/12/23", + "verification": [], "instances": [ "Python/3.9 aiohttp/3.7.3", "Python/3.8 aiohttp/3.7.2", @@ -303,6 +423,7 @@ ], "pattern": "httpx", "addition_date": "2019/12/23", + "verification": [], "instances": [ "python-httpx/0.16.1", "python-httpx/0.13.0.dev1" @@ -315,6 +436,7 @@ "programmatic" ], "pattern": "libwww-perl", + "verification": [], "instances": [ "2Bone_LinkChecker/1.0 libwww-perl/6.03", "2Bone_LinkChkr/1.0 libwww-perl/6.03", @@ -327,6 +449,7 @@ "programmatic" ], "pattern": "httpunit", + "verification": [], "instances": [ "httpunit/1.x" ] @@ -337,6 +460,7 @@ "tool" ], "pattern": "Nutch", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.16 (KHTML, like Gecko; compatible; Friendly_Crawler/2.0) Chrome/120.0.6099.217 Safari/605.1.15/Nutch-1.20-SNAPSHOT", "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", @@ -351,6 +475,7 @@ "pattern": "Go-http-client", "addition_date": "2016/03/26", "url": "https://golang.org/pkg/net/http/", + "verification": [], "instances": [ "Go-http-client/1.1", "Go-http-client/2.0" @@ -364,6 +489,7 @@ "pattern": "phpcrawl", "addition_date": "2012/09/17", "url": "http://phpcrawl.cuab.de/", + "verification": [], "instances": [ "phpcrawl" ] @@ -376,6 +502,7 @@ ], "pattern": "msnbot", "url": "http://search.msn.com/msnbot.htm", + "verification": [], "instances": [ "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)", "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)", @@ -399,6 +526,7 @@ "unknown" ], "pattern": "jyxobot", + "verification": [], "instances": [] }, { @@ -407,6 +535,7 @@ "unknown" ], "pattern": "FAST-WebCrawler", + "verification": [], "instances": [ "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", @@ -420,6 +549,7 @@ "unknown" ], "pattern": "FAST Enterprise Crawler", + "verification": [], "instances": [ "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)" @@ -431,6 +561,7 @@ "unknown" ], "pattern": "BIGLOTRON", + "verification": [], "instances": [ "BIGLOTRON (Beta 2;GNU/Linux)" ] @@ -441,6 +572,7 @@ "search-engine" ], "pattern": "Teoma", + "verification": [], "instances": [ "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" @@ -453,6 +585,7 @@ "unknown" ], "pattern": "convera", + "verification": [], "instances": [ "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" ], @@ -464,6 +597,7 @@ "unknown" ], "pattern": "seekbot", + "verification": [], "instances": [ "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2" ], @@ -475,6 +609,7 @@ "unknown" ], "pattern": "Gigabot", + "verification": [], "instances": [ "Gigabot/1.0", "Gigabot/2.0 (http://www.gigablast.com/spider.html)" @@ -488,6 +623,7 @@ "programmatic" ], "pattern": "Gigablast", + "verification": [], "instances": [ "GigablastOpenSource/1.0" ], @@ -499,6 +635,7 @@ "amazon" ], "pattern": "exabot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)", "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)", @@ -514,6 +651,7 @@ "unknown" ], "pattern": "ia_archiver", + "verification": [], "instances": [ "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", "ia_archiver-web.archive.org" @@ -525,6 +663,7 @@ "unknown" ], "pattern": "GingerCrawler", + "verification": [], "instances": [ "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)" ] @@ -535,6 +674,7 @@ "unknown" ], "pattern": "webmon ", + "verification": [], "instances": [] }, { @@ -543,6 +683,7 @@ "unknown" ], "pattern": "HTTrack", + "verification": [], "instances": [ "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" ] @@ -553,6 +694,7 @@ "unknown" ], "pattern": "grub\\.org", + "verification": [], "instances": [ "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", @@ -573,6 +715,7 @@ "unknown" ], "pattern": "UsineNouvelleCrawler", + "verification": [], "instances": [] }, { @@ -581,6 +724,7 @@ "unknown" ], "pattern": "antibot", + "verification": [], "instances": [] }, { @@ -589,6 +733,7 @@ "unknown" ], "pattern": "netresearchserver", + "verification": [], "instances": [] }, { @@ -597,6 +742,7 @@ "search-engine" ], "pattern": "speedy", + "verification": [], "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", @@ -611,6 +757,7 @@ "unknown" ], "pattern": "fluffy", + "verification": [], "instances": [] }, { @@ -619,6 +766,7 @@ "academic" ], "pattern": "findlink", + "verification": [], "instances": [ "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", @@ -651,6 +799,7 @@ "microsoft" ], "pattern": "msrbot", + "verification": [], "instances": [] }, { @@ -659,6 +808,7 @@ "unknown" ], "pattern": "panscient", + "verification": [], "instances": [ "panscient.com" ] @@ -669,6 +819,7 @@ "search-engine" ], "pattern": "yacybot", + "verification": [], "instances": [ "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", @@ -725,6 +876,7 @@ "ai" ], "pattern": "AISearchBot", + "verification": [], "instances": [] }, { @@ -733,6 +885,7 @@ "unknown" ], "pattern": "ips-agent", + "verification": [], "instances": [ "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", @@ -747,6 +900,7 @@ "unknown" ], "pattern": "tagoobot", + "verification": [], "instances": [] }, { @@ -755,6 +909,7 @@ "search-engine" ], "pattern": "MJ12bot", + "verification": [], "instances": [ "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", @@ -784,6 +939,7 @@ "unknown" ], "pattern": "woriobot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; woriobot +http://worio.com)", "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)" @@ -795,6 +951,7 @@ "unknown" ], "pattern": "yanga", + "verification": [], "instances": [ "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" ] @@ -805,6 +962,7 @@ "unknown" ], "pattern": "buzzbot", + "verification": [], "instances": [ "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)" ] @@ -815,6 +973,7 @@ "unknown" ], "pattern": "mlbot", + "verification": [], "instances": [ "MLBot (www.metadatalabs.com/mlbot)" ] @@ -826,6 +985,16 @@ ], "pattern": "yandex\\.com\\/bots", "url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs", + "verification": [ + { + "type": "dns", + "masks": [ + "@.yandex.ru", + "@.yandex.com", + "@.yandex.net" + ] + } + ], "instances": [ "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)", @@ -878,6 +1047,7 @@ ], "pattern": "purebot", "addition_date": "2010/01/19", + "verification": [], "instances": [] }, { @@ -888,6 +1058,7 @@ "pattern": "Linguee Bot", "addition_date": "2010/01/26", "url": "http://www.linguee.com/bot", + "verification": [], "instances": [ "Linguee Bot (http://www.linguee.com/bot)", "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)" @@ -901,6 +1072,7 @@ "pattern": "CyberPatrol", "addition_date": "2010/02/11", "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", + "verification": [], "instances": [ "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" ] @@ -912,6 +1084,7 @@ ], "pattern": "voilabot", "addition_date": "2010/05/18", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)" @@ -925,6 +1098,15 @@ "pattern": "Baiduspider", "addition_date": "2010/07/15", "url": "http://www.baidu.jp/spider/", + "verification": [ + { + "type": "dns", + "masks": [ + "@.baidu.jp", + "@.baidu.com" + ] + } + ], "instances": [ "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)" @@ -937,6 +1119,7 @@ ], "pattern": "citeseerxbot", "addition_date": "2010/07/17", + "verification": [], "instances": [] }, { @@ -947,6 +1130,7 @@ "pattern": "spbot", "addition_date": "2010/07/31", "url": "http://www.seoprofiler.com/bot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", @@ -992,6 +1176,7 @@ "pattern": "twengabot", "addition_date": "2010/08/03", "url": "http://www.twenga.com/bot.html", + "verification": [], "instances": [] }, { @@ -1002,6 +1187,7 @@ "pattern": "postrank", "addition_date": "2010/08/03", "url": "http://www.postrank.com", + "verification": [], "instances": [ "PostRank/2.0 (postrank.com)", "PostRank/2.0 (postrank.com; 1 subscribers)" @@ -1016,6 +1202,7 @@ "pattern": "Turnitin", "addition_date": "2010/09/26", "url": "http://www.turnitin.com", + "verification": [], "instances": [ "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", "Turnitin (https://bit.ly/2UvnfoQ)" @@ -1029,6 +1216,7 @@ "pattern": "scribdbot", "addition_date": "2010/09/28", "url": "http://www.scribd.com", + "verification": [], "instances": [] }, { @@ -1039,6 +1227,7 @@ "pattern": "page2rss", "addition_date": "2010/10/07", "url": "http://www.page2rss.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)" ] @@ -1051,6 +1240,7 @@ "pattern": "sitebot", "addition_date": "2010/12/15", "url": "http://www.sitebot.org", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)" ] @@ -1063,6 +1253,7 @@ "pattern": "linkdex", "addition_date": "2011/01/06", "url": "http://www.linkdex.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", @@ -1081,6 +1272,7 @@ ], "pattern": "Adidxbot", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", + "verification": [], "instances": [] }, { @@ -1091,6 +1283,7 @@ "pattern": "ezooms", "addition_date": "2011/04/27", "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" ] @@ -1102,6 +1295,7 @@ ], "pattern": "dotbot", "addition_date": "2011/04/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)", "dotbot" @@ -1114,6 +1308,7 @@ ], "pattern": "Mail\\.RU_Bot", "addition_date": "2011/04/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/", @@ -1129,6 +1324,7 @@ "pattern": "discobot", "addition_date": "2011/05/03", "url": "http://discoveryengine.com/discobot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)", "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)", @@ -1144,6 +1340,7 @@ "pattern": "heritrix", "addition_date": "2011/06/21", "url": "https://github.com/internetarchive/heritrix3/wiki", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", @@ -1177,6 +1374,7 @@ "pattern": "findthatfile", "addition_date": "2011/06/21", "url": "http://www.findthatfile.com/", + "verification": [], "instances": [] }, { @@ -1187,6 +1385,7 @@ "pattern": "europarchive\\.org", "addition_date": "2011/06/21", "url": "", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" ] @@ -1199,6 +1398,7 @@ "pattern": "NerdByNature\\.Bot", "addition_date": "2011/07/12", "url": "http://www.nerdbynature.net/bot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)" ] @@ -1211,6 +1411,7 @@ "pattern": "(sistrix|SISTRIX) [cC]rawler", "addition_date": "2011/08/02", "url": "https://www.sistrix.com/tutorials/crawling-errors-in-the-optimizer/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)" ] @@ -1222,6 +1423,7 @@ ], "pattern": "Ahrefs(Bot|SiteAudit)", "addition_date": "2011/08/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)", @@ -1239,6 +1441,7 @@ ], "pattern": "fuelbot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "fuelbot" ] @@ -1250,6 +1453,7 @@ ], "pattern": "CrunchBot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)" ] @@ -1261,6 +1465,7 @@ ], "pattern": "IndeedBot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" ] @@ -1272,6 +1477,7 @@ ], "pattern": "mappydata", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" ] @@ -1283,6 +1489,7 @@ ], "pattern": "woobot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "woobot" ] @@ -1294,6 +1501,7 @@ ], "pattern": "ZoominfoBot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "ZoominfoBot (zoominfobot at zoominfo dot com)" ] @@ -1305,6 +1513,7 @@ ], "pattern": "PrivacyAwareBot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)" ] @@ -1316,6 +1525,7 @@ ], "pattern": "Multiviewbot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot" ] @@ -1327,6 +1537,7 @@ ], "pattern": "SWIMGBot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot" ] @@ -1338,6 +1549,7 @@ ], "pattern": "Grobbot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)" ] @@ -1349,6 +1561,7 @@ ], "pattern": "eright", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" ] @@ -1360,6 +1573,7 @@ ], "pattern": "Apercite", "addition_date": "2018/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" ] @@ -1371,6 +1585,7 @@ ], "pattern": "semanticbot", "addition_date": "2018/06/28", + "verification": [], "instances": [ "semanticbot", "semanticbot (info@semanticaudience.com)" @@ -1384,6 +1599,7 @@ "pattern": "Aboundex", "addition_date": "2011/09/28", "url": "http://www.aboundex.com/crawler/", + "verification": [], "instances": [ "Aboundex/0.2 (http://www.aboundex.com/crawler/)", "Aboundex/0.3 (http://www.aboundex.com/crawler/)" @@ -1396,6 +1612,7 @@ ], "pattern": "domaincrawler", "addition_date": "2011/10/21", + "verification": [], "instances": [ "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)" ] @@ -1408,6 +1625,7 @@ "pattern": "wbsearchbot", "addition_date": "2011/12/21", "url": "http://www.warebay.com/bot.html", + "verification": [], "instances": [] }, { @@ -1418,6 +1636,7 @@ "pattern": "summify", "addition_date": "2012/01/04", "url": "http://summify.com", + "verification": [], "instances": [ "Summify (Summify/1.0.1; +http://summify.com)" ] @@ -1432,6 +1651,7 @@ "pattern": "CCBot", "addition_date": "2012/02/05", "url": "http://www.commoncrawl.org/bot.html", + "verification": [], "instances": [ "CCBot/2.0 (http://commoncrawl.org/faq/)", "CCBot/2.0 (https://commoncrawl.org/faq/)" @@ -1444,6 +1664,7 @@ ], "pattern": "edisterbot", "addition_date": "2012/02/25", + "verification": [], "instances": [] }, { @@ -1453,6 +1674,7 @@ ], "pattern": "SeznamBot", "addition_date": "2012/03/14", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", @@ -1469,6 +1691,7 @@ ], "pattern": "ec2linkfinder", "addition_date": "2012/03/22", + "verification": [], "instances": [ "ec2linkfinder" ] @@ -1480,6 +1703,7 @@ ], "pattern": "gslfbot", "addition_date": "2012/04/03", + "verification": [], "instances": [] }, { @@ -1489,6 +1713,7 @@ ], "pattern": "aiHitBot", "addition_date": "2012/04/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" ] @@ -1500,6 +1725,7 @@ ], "pattern": "intelium_bot", "addition_date": "2012/05/07", + "verification": [], "instances": [] }, { @@ -1512,6 +1738,7 @@ ], "pattern": "facebookexternalhit", "addition_date": "2012/05/07", + "verification": [], "instances": [ "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", "facebookexternalhit/1.1", @@ -1527,6 +1754,7 @@ "pattern": "Yeti", "addition_date": "2012/05/07", "url": "http://naver.me/bot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" ] @@ -1538,6 +1766,7 @@ ], "pattern": "RetrevoPageAnalyzer", "addition_date": "2012/05/07", + "verification": [], "instances": [ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" ] @@ -1549,6 +1778,7 @@ ], "pattern": "lb-spider", "addition_date": "2012/05/07", + "verification": [], "instances": [] }, { @@ -1559,6 +1789,7 @@ "pattern": "Sogou", "addition_date": "2012/05/13", "url": "http://www.sogou.com/docs/help/webmasters.htm#07", + "verification": [], "instances": [ "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", @@ -1573,6 +1804,7 @@ "pattern": "lssbot", "addition_date": "2012/05/15", "url": "https://www.lssbot.com/", + "verification": [], "instances": [] }, { @@ -1583,6 +1815,7 @@ "pattern": "careerbot", "addition_date": "2012/05/23", "url": "http://www.career-x.de/bot.html", + "verification": [], "instances": [] }, { @@ -1593,6 +1826,7 @@ "pattern": "wotbox", "addition_date": "2012/06/12", "url": "http://www.wotbox.com", + "verification": [], "instances": [ "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", "Wotbox/2.01 (+http://www.wotbox.com/bot/)" @@ -1606,6 +1840,7 @@ "pattern": "wocbot", "addition_date": "2012/07/25", "url": "http://www.wocodi.com/crawler", + "verification": [], "instances": [] }, { @@ -1616,6 +1851,7 @@ "pattern": "ichiro", "addition_date": "2012/08/28", "url": "http://help.goo.ne.jp/help/article/1142", + "verification": [], "instances": [ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", @@ -1642,6 +1878,7 @@ "pattern": "DuckDuckBot", "addition_date": "2012/09/19", "url": "http://duckduckgo.com/duckduckbot.html", + "verification": [], "instances": [ "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", @@ -1656,6 +1893,7 @@ ], "pattern": "lssrocketcrawler", "addition_date": "2012/09/24", + "verification": [], "instances": [] }, { @@ -1666,6 +1904,7 @@ "pattern": "drupact", "addition_date": "2012/09/27", "url": "http://www.arocom.de/drupact", + "verification": [], "instances": [ "drupact/0.7; http://www.arocom.de/drupact" ] @@ -1677,6 +1916,7 @@ ], "pattern": "webcompanycrawler", "addition_date": "2012/10/03", + "verification": [], "instances": [] }, { @@ -1687,6 +1927,7 @@ "pattern": "acoonbot", "addition_date": "2012/10/07", "url": "http://www.acoon.de/robot.asp", + "verification": [], "instances": [] }, { @@ -1697,6 +1938,7 @@ "pattern": "openindexspider", "addition_date": "2012/10/26", "url": "http://www.openindex.io/en/webmasters/spider.html", + "verification": [], "instances": [] }, { @@ -1706,6 +1948,7 @@ ], "pattern": "gnam gnam spider", "addition_date": "2012/10/31", + "verification": [], "instances": [] }, { @@ -1714,6 +1957,7 @@ "archive" ], "pattern": "web-archive-net\\.com\\.bot", + "verification": [], "instances": [] }, { @@ -1724,6 +1968,7 @@ "pattern": "backlinkcrawler", "addition_date": "2013/01/04", "url": "http://www.backlinktest.com/crawler.html", + "verification": [], "instances": [] }, { @@ -1734,6 +1979,7 @@ "pattern": "coccoc", "addition_date": "2013/01/04", "url": "http://help.coccoc.vn/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", @@ -1756,6 +2002,7 @@ "pattern": "integromedb", "addition_date": "2013/01/10", "url": "http://www.integromedb.org/Crawler", + "verification": [], "instances": [ "www.integromedb.org/Crawler" ] @@ -1767,6 +2014,7 @@ ], "pattern": "content crawler spider", "addition_date": "2013/01/11", + "verification": [], "instances": [] }, { @@ -1776,6 +2024,7 @@ ], "pattern": "toplistbot", "addition_date": "2013/02/05", + "verification": [], "instances": [] }, { @@ -1785,6 +2034,7 @@ ], "pattern": "it2media-domain-crawler", "addition_date": "2013/03/12", + "verification": [], "instances": [ "it2media-domain-crawler/1.0 on crawler-prod.it2media.de", "it2media-domain-crawler/2.0" @@ -1797,6 +2047,7 @@ ], "pattern": "ip-web-crawler\\.com", "addition_date": "2013/03/22", + "verification": [], "instances": [] }, { @@ -1806,6 +2057,7 @@ ], "pattern": "siteexplorer\\.info", "addition_date": "2013/05/01", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" @@ -1818,6 +2070,7 @@ ], "pattern": "elisabot", "addition_date": "2013/06/27", + "verification": [], "instances": [] }, { @@ -1828,6 +2081,7 @@ "pattern": "proximic", "addition_date": "2013/09/12", "url": "http://www.proximic.com/info/spider.php", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" @@ -1841,6 +2095,7 @@ "pattern": "changedetection", "addition_date": "2013/09/13", "url": "https://visualping.io/", + "verification": [], "instances": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" ] @@ -1852,6 +2107,7 @@ ], "pattern": "arabot", "addition_date": "2013/10/09", + "verification": [], "instances": [] }, { @@ -1861,6 +2117,7 @@ ], "pattern": "WeSEE:Search", "addition_date": "2013/11/18", + "verification": [], "instances": [ "WeSEE:Search", "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" @@ -1873,6 +2130,7 @@ ], "pattern": "niki-bot", "addition_date": "2014/01/01", + "verification": [], "instances": [] }, { @@ -1883,6 +2141,7 @@ "pattern": "CrystalSemanticsBot", "addition_date": "2014/02/17", "url": "http://www.crystalsemantics.com/user-agent/", + "verification": [], "instances": [] }, { @@ -1893,6 +2152,7 @@ "pattern": "rogerbot", "addition_date": "2014/02/28", "url": "https://moz.com/help/moz-procedures/crawlers/rogerbot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)", @@ -1917,6 +2177,7 @@ "pattern": "360Spider", "addition_date": "2014/03/14", "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", @@ -1938,6 +2199,7 @@ "pattern": "psbot", "addition_date": "2014/03/31", "url": "http://www.picsearch.com/bot.html", + "verification": [], "instances": [ "psbot-image (+http://www.picsearch.com/bot.html)", "psbot-page (+http://www.picsearch.com/bot.html)", @@ -1952,6 +2214,7 @@ "pattern": "InterfaxScanBot", "addition_date": "2014/03/31", "url": "http://scan-interfax.ru", + "verification": [], "instances": [] }, { @@ -1962,6 +2225,7 @@ "pattern": "CC Metadata Scaper", "addition_date": "2014/04/01", "url": "http://wiki.creativecommons.org/Metadata_Scraper", + "verification": [], "instances": [ "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" ] @@ -1974,6 +2238,7 @@ "pattern": "g00g1e\\.net", "addition_date": "2014/04/01", "url": "http://www.g00g1e.net/", + "verification": [], "instances": [] }, { @@ -1984,6 +2249,7 @@ "pattern": "GrapeshotCrawler", "addition_date": "2014/04/01", "url": "http://www.grapeshot.co.uk/crawler.php", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)" ] @@ -1996,6 +2262,7 @@ "pattern": "urlappendbot", "addition_date": "2014/05/10", "url": "http://www.profound.net/urlappendbot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)" ] @@ -2007,6 +2274,7 @@ ], "pattern": "brainobot", "addition_date": "2014/06/24", + "verification": [], "instances": [] }, { @@ -2016,6 +2284,7 @@ ], "pattern": "fr-crawler", "addition_date": "2014/07/31", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; fr-crawler/1.1)" ] @@ -2027,6 +2296,7 @@ ], "pattern": "binlar", "addition_date": "2014/09/12", + "verification": [], "instances": [ "binlar_2.6.3 binlar2.6.3@unspecified.mail", "binlar_2.6.3 binlar_2.6.3@unspecified.mail", @@ -2042,6 +2312,7 @@ ], "pattern": "SimpleCrawler", "addition_date": "2014/09/12", + "verification": [], "instances": [ "SimpleCrawler/0.1" ] @@ -2055,6 +2326,7 @@ "pattern": "Twitterbot", "addition_date": "2014/09/12", "url": "https://dev.twitter.com/cards/getting-started", + "verification": [], "instances": [ "Twitterbot/0.1", "Twitterbot/1.0" @@ -2067,6 +2339,7 @@ ], "pattern": "cXensebot", "addition_date": "2014/10/05", + "verification": [], "instances": [ "cXensebot/1.1a" ], @@ -2079,6 +2352,7 @@ ], "pattern": "smtbot", "addition_date": "2014/10/04", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", "SMTBot (similartech.com/smtbot)", @@ -2096,6 +2370,7 @@ "pattern": "bnf\\.fr_bot", "addition_date": "2014/11/18", "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)", "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)" @@ -2109,6 +2384,7 @@ "pattern": "A6-Indexer", "addition_date": "2014/12/05", "url": "http://www.a6corp.com/a6-web-scraping-policy/", + "verification": [], "instances": [ "A6-Indexer" ] @@ -2121,6 +2397,7 @@ "pattern": "ADmantX", "addition_date": "2014/12/05", "url": "http://www.admantx.com", + "verification": [], "instances": [ "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" ] @@ -2135,6 +2412,7 @@ "pattern": "Face(book){0,1}[Bb]ot", "url": "https://developers.facebook.com/docs/sharing/bot", "addition_date": "2014/12/30", + "verification": [], "instances": [ "Facebot/1.0", "Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)" @@ -2146,6 +2424,7 @@ "unknown" ], "pattern": "OrangeBot\\/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com" ], @@ -2158,6 +2437,7 @@ ], "pattern": "memorybot", "url": "http://mignify.com/bot.htm", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)" ], @@ -2170,6 +2450,7 @@ ], "pattern": "AdvBot", "url": "http://advbot.net/bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)" ], @@ -2182,6 +2463,7 @@ ], "pattern": "MegaIndex", "url": "https://www.megaindex.ru/?tab=linkAnalyze", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" @@ -2196,6 +2478,7 @@ ], "pattern": "SemanticScholarBot", "url": "https://www.semanticscholar.org/crawler", + "verification": [], "instances": [ "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)", "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)" @@ -2209,6 +2492,7 @@ ], "pattern": "ltx71", "url": "http://ltx71.com/", + "verification": [], "instances": [ "ltx71 - (http://ltx71.com/)" ], @@ -2221,6 +2505,7 @@ ], "pattern": "nerdybot", "url": "http://nerdybot.com/", + "verification": [], "instances": [ "nerdybot" ], @@ -2233,6 +2518,7 @@ ], "pattern": "xovibot", "url": "http://www.xovibot.net/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" ], @@ -2246,6 +2532,7 @@ ], "pattern": "BUbiNG", "url": "http://law.di.unimi.it/BUbiNG.html", + "verification": [], "instances": [ "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" ], @@ -2258,6 +2545,7 @@ ], "pattern": "Qwantify", "url": "https://www.qwant.com/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", @@ -2276,6 +2564,7 @@ "depends_on": [ "heritrix" ], + "verification": [], "instances": [ "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", @@ -2294,6 +2583,7 @@ "pattern": "Applebot", "url": "http://www.apple.com/go/applebot", "addition_date": "2015/04/15", + "verification": [], "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", @@ -2309,6 +2599,7 @@ ], "pattern": "i[Tt][Mm][Ss]", "addition_date": "2024/09/19", + "verification": [], "instances": [ "iTMS", "itms" @@ -2322,6 +2613,7 @@ ], "pattern": "TweetmemeBot", "url": "http://datasift.com/bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" ], @@ -2334,6 +2626,7 @@ ], "pattern": "crawler4j", "url": "https://github.com/yasserg/crawler4j", + "verification": [], "instances": [ "crawler4j (http://code.google.com/p/crawler4j/)", "crawler4j (https://github.com/yasserg/crawler4j/)" @@ -2347,6 +2640,7 @@ ], "pattern": "findxbot", "url": "http://www.findxbot.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)" ], @@ -2359,6 +2653,7 @@ ], "pattern": "S[eE][mM]rushBot", "url": "http://www.semrush.com/bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", @@ -2378,6 +2673,7 @@ ], "pattern": "yoozBot", "url": "http://yooz.ir", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)" ], @@ -2390,6 +2686,7 @@ ], "pattern": "lipperhey", "url": "http://www.lipperhey.com/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", @@ -2405,6 +2702,7 @@ ], "pattern": "Y!J", "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6", + "verification": [], "instances": [ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)", "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", @@ -2422,6 +2720,7 @@ ], "pattern": "Domain Re-Animator Bot", "url": "http://domainreanimator.com", + "verification": [], "instances": [ "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com" ], @@ -2434,6 +2733,7 @@ ], "pattern": "AddThis", "url": "https://www.addthis.com", + "verification": [], "instances": [ "AddThis.com robot tech.support@clearspring.com" ], @@ -2446,6 +2746,7 @@ ], "pattern": "Screaming Frog SEO Spider", "url": "http://www.screamingfrog.co.uk/seo-spider", + "verification": [], "instances": [ "Screaming Frog SEO Spider/5.1" ], @@ -2458,6 +2759,7 @@ ], "pattern": "MetaURI", "url": "http://www.useragentstring.com/MetaURI_id_17683.php", + "verification": [], "instances": [ "MetaURI API/2.0 +metauri.com" ], @@ -2471,6 +2773,7 @@ ], "pattern": "Scrapy", "url": "http://scrapy.org/", + "verification": [], "instances": [ "Scrapy/1.0.3 (+http://scrapy.org)" ], @@ -2483,6 +2786,7 @@ ], "pattern": "Livelap[bB]ot", "url": "http://site.livelap.com/crawler", + "verification": [], "instances": [ "LivelapBot/0.2 (http://site.livelap.com/crawler)", "Livelapbot/0.1" @@ -2496,6 +2800,7 @@ ], "pattern": "OpenHoseBot", "url": "http://www.openhose.org/bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)" ], @@ -2508,6 +2813,7 @@ ], "pattern": "CapsuleChecker", "url": "http://www.capsulink.com/about", + "verification": [], "instances": [ "CapsuleChecker (http://www.capsulink.com/)" ], @@ -2520,6 +2826,7 @@ ], "pattern": "collection@infegy\\.com", "url": "http://infegy.com/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" ], @@ -2532,6 +2839,7 @@ ], "pattern": "IstellaBot", "url": "http://www.tiscali.it/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)" ], @@ -2545,6 +2853,7 @@ "pattern": "DeuSu\\/", "addition_date": "2016/01/23", "url": "https://deusu.de/robot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" @@ -2557,6 +2866,7 @@ ], "pattern": "betaBot", "addition_date": "2016/01/23", + "verification": [], "instances": [] }, { @@ -2567,6 +2877,7 @@ "pattern": "Cliqzbot\\/", "addition_date": "2016/01/23", "url": "http://cliqz.com/company/cliqzbot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)", "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", @@ -2583,6 +2894,7 @@ "pattern": "MojeekBot\\/", "addition_date": "2016/01/23", "url": "https://www.mojeek.com/bot.html", + "verification": [], "instances": [ "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", @@ -2600,6 +2912,7 @@ "pattern": "netEstate NE Crawler", "addition_date": "2016/01/23", "url": "http://www.website-datenbank.de/", + "verification": [], "instances": [ "netEstate NE Crawler (+http://www.sengine.info/)", "netEstate NE Crawler (+http://www.website-datenbank.de/)" @@ -2613,6 +2926,7 @@ "pattern": "SafeSearch microdata crawler", "addition_date": "2016/01/23", "url": "https://safesearch.avira.com", + "verification": [], "instances": [ "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)" ] @@ -2625,6 +2939,7 @@ "pattern": "Gluten Free Crawler\\/", "addition_date": "2016/01/23", "url": "http://glutenfreepleasure.com/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)" ] @@ -2637,6 +2952,7 @@ "pattern": "Sonic", "addition_date": "2016/02/08", "url": "https://www.yama.info.waseda.ac.jp/~crawler/info_en.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", @@ -2651,6 +2967,7 @@ "pattern": "Sysomos", "addition_date": "2016/02/08", "url": "http://www.sysomos.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" ] @@ -2663,6 +2980,7 @@ "pattern": "Trove", "addition_date": "2016/02/08", "url": "http://www.trove.com", + "verification": [], "instances": [] }, { @@ -2673,6 +2991,7 @@ "pattern": "deadlinkchecker", "addition_date": "2016/02/08", "url": "http://www.deadlinkchecker.com", + "verification": [], "instances": [ "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", "www.deadlinkchecker.com XMLHTTP/1.0", @@ -2688,6 +3007,7 @@ "pattern": "Slack-ImgProxy", "addition_date": "2016/04/25", "url": "https://api.slack.com/robots", + "verification": [], "instances": [ "Slack-ImgProxy (+https://api.slack.com/robots)", "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", @@ -2705,6 +3025,7 @@ "pattern": "Embedly", "addition_date": "2016/04/25", "url": "http://support.embed.ly", + "verification": [], "instances": [ "Embedly +support@embed.ly", "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", @@ -2719,6 +3040,7 @@ "pattern": "RankActiveLinkBot", "addition_date": "2016/06/20", "url": "https://rankactive.com/resources/rankactive-linkbot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)" ] @@ -2731,6 +3053,7 @@ "pattern": "iskanie", "addition_date": "2016/09/02", "url": "http://www.iskanie.com", + "verification": [], "instances": [ "iskanie (+http://www.iskanie.com)" ] @@ -2744,6 +3067,7 @@ "pattern": "SafeDNSBot", "addition_date": "2016/09/10", "url": "https://www.safedns.com/searchbot", + "verification": [], "instances": [ "SafeDNSBot (https://www.safedns.com/searchbot)" ] @@ -2756,6 +3080,7 @@ ], "pattern": "SkypeUriPreview", "addition_date": "2016/10/10", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" ] @@ -2768,6 +3093,7 @@ "pattern": "Veoozbot", "addition_date": "2016/11/03", "url": "http://www.veooz.com/veoozbot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)" ] @@ -2780,6 +3106,7 @@ "pattern": "Slackbot", "addition_date": "2016/11/03", "url": "https://api.slack.com/robots", + "verification": [], "instances": [ "Slackbot-LinkExpanding (+https://api.slack.com/robots)", "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", @@ -2794,6 +3121,7 @@ "pattern": "redditbot", "addition_date": "2016/11/03", "url": "http://www.reddit.com/feedback", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)" ] @@ -2806,6 +3134,7 @@ "pattern": "datagnionbot", "addition_date": "2016/11/03", "url": "http://www.datagnion.com/bot.html", + "verification": [], "instances": [ "datagnionbot (+http://www.datagnion.com/bot.html)" ] @@ -2819,6 +3148,7 @@ "pattern": "Google-Adwords-Instant", "addition_date": "2016/11/03", "url": "http://www.google.com/adsbot.html", + "verification": [], "instances": [ "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" ] @@ -2830,6 +3160,7 @@ ], "pattern": "adbeat_bot", "addition_date": "2016/11/04", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", "adbeat_bot" @@ -2845,6 +3176,7 @@ "pattern": "WhatsApp", "addition_date": "2016/11/15", "url": "https://www.whatsapp.com/", + "verification": [], "instances": [ "WhatsApp", "WhatsApp/0.3.4479 N", @@ -2880,6 +3212,7 @@ ], "pattern": "contxbot", "addition_date": "2017/02/25", + "verification": [], "instances": [ "Mozilla/5.0 (compatible;contxbot/1.0)" ] @@ -2891,6 +3224,7 @@ ], "pattern": "pinterest\\.com\\/bot", "addition_date": "2017/03/03", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", "Pinterest/0.2 (+http://www.pinterest.com/bot.html)" @@ -2904,6 +3238,7 @@ ], "pattern": "electricmonk", "addition_date": "2017/03/04", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" ], @@ -2916,6 +3251,7 @@ ], "pattern": "GarlikCrawler", "addition_date": "2017/03/18", + "verification": [], "instances": [ "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)" ], @@ -2930,6 +3266,7 @@ "pattern": "BingPreview\\/", "addition_date": "2017/04/23", "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", @@ -2945,6 +3282,7 @@ ], "pattern": "vebidoobot", "addition_date": "2017/05/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/" ], @@ -2957,6 +3295,7 @@ ], "pattern": "FemtosearchBot", "addition_date": "2017/05/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)" ], @@ -2970,6 +3309,7 @@ ], "pattern": "Yahoo Link Preview", "addition_date": "2017/06/28", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)" ], @@ -2982,6 +3322,7 @@ ], "pattern": "MetaJobBot", "addition_date": "2017/08/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)" ], @@ -2994,6 +3335,7 @@ ], "pattern": "DomainStatsBot", "addition_date": "2017/08/16", + "verification": [], "instances": [ "DomainStatsBot/1.0 (http://domainstats.io/our-bot)" ], @@ -3006,6 +3348,7 @@ ], "pattern": "mindUpBot", "addition_date": "2017/08/16", + "verification": [], "instances": [ "mindUpBot (datenbutler.de)" ], @@ -3018,6 +3361,7 @@ ], "pattern": "Daum\\/", "addition_date": "2017/08/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" ], @@ -3030,6 +3374,7 @@ ], "pattern": "Jugendschutzprogramm-Crawler", "addition_date": "2017/08/16", + "verification": [], "instances": [ "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" ], @@ -3043,6 +3388,7 @@ ], "pattern": "Xenu Link Sleuth", "addition_date": "2017/08/19", + "verification": [], "instances": [ "Xenu Link Sleuth/1.3.8" ], @@ -3055,6 +3401,7 @@ ], "pattern": "Pcore-HTTP", "addition_date": "2017/08/19", + "verification": [], "instances": [ "Pcore-HTTP/v0.40.3", "Pcore-HTTP/v0.44.0" @@ -3068,6 +3415,7 @@ ], "pattern": "moatbot", "addition_date": "2017/09/16", + "verification": [], "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot" @@ -3081,6 +3429,7 @@ ], "pattern": "KosmioBot", "addition_date": "2017/09/16", + "verification": [], "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)" ], @@ -3093,6 +3442,7 @@ ], "pattern": "[pP]ingdom", "addition_date": "2017/09/16", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)", @@ -3114,6 +3464,7 @@ ], "pattern": "AppInsights", "addition_date": "2019/03/09", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)" ], @@ -3126,6 +3477,7 @@ ], "pattern": "PhantomJS", "addition_date": "2017/09/18", + "verification": [], "instances": [ "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" ], @@ -3138,6 +3490,7 @@ ], "pattern": "Gowikibot", "addition_date": "2017/10/26", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)" ], @@ -3150,6 +3503,7 @@ ], "pattern": "PiplBot", "addition_date": "2017/10/30", + "verification": [], "instances": [ "PiplBot (+http://www.pipl.com/bot/)", "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)" @@ -3165,6 +3519,7 @@ "pattern": "Discordbot", "addition_date": "2017/09/22", "url": "https://discordapp.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" ] @@ -3177,6 +3532,7 @@ ], "pattern": "TelegramBot", "addition_date": "2017/10/01", + "verification": [], "instances": [ "TelegramBot (like TwitterBot)" ] @@ -3189,6 +3545,7 @@ "pattern": "Jetslide", "addition_date": "2017/09/27", "url": "http://jetsli.de/crawler", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" ] @@ -3201,6 +3558,7 @@ "pattern": "newsharecounts", "addition_date": "2017/09/30", "url": "http://newsharecounts.com/crawler", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" ] @@ -3213,6 +3571,7 @@ "pattern": "James BOT", "addition_date": "2017/10/12", "url": "http://cognitiveseo.com/bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" ] @@ -3225,6 +3584,7 @@ "pattern": "Bark[rR]owler", "addition_date": "2017/10/09", "url": "http://www.exensa.com/crawl", + "verification": [], "instances": [ "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", "Barkrowler/0.7 (+http://www.exensa.com/crawl)", @@ -3240,6 +3600,7 @@ "pattern": "TinEye", "addition_date": "2017/10/14", "url": "http://www.tineye.com/crawler.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", "TinEye/1.1 (http://tineye.com/crawler.html)" @@ -3253,6 +3614,7 @@ "pattern": "SocialRankIOBot", "addition_date": "2017/10/19", "url": "http://socialrank.io/about", + "verification": [], "instances": [ "SocialRankIOBot; http://socialrank.io/about" ] @@ -3265,6 +3627,7 @@ "pattern": "trendictionbot", "addition_date": "2017/10/30", "url": "http://www.trendiction.de/bot", + "verification": [], "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0" @@ -3277,6 +3640,7 @@ ], "pattern": "Ocarinabot", "addition_date": "2017/09/27", + "verification": [], "instances": [ "Ocarinabot" ] @@ -3289,6 +3653,7 @@ "pattern": "epicbot", "addition_date": "2017/10/31", "url": "http://www.epictions.com/epicbot", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)" ] @@ -3301,6 +3666,7 @@ "pattern": "Primalbot", "addition_date": "2017/09/27", "url": "https://www.primal.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)" ] @@ -3314,6 +3680,7 @@ "pattern": "DuckDuckGo-Favicons-Bot", "addition_date": "2017/10/06", "url": "http://duckduckgo.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" ] @@ -3326,6 +3693,7 @@ "pattern": "GnowitNewsbot", "addition_date": "2017/10/30", "url": "http://www.gnowit.com", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com" ] @@ -3338,6 +3706,7 @@ "pattern": "Leikibot", "addition_date": "2017/09/24", "url": "http://www.leiki.com", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)" ] @@ -3351,6 +3720,7 @@ "pattern": "LinkArchiver", "addition_date": "2017/09/24", "url": "https://github.com/thisisparker/linkarchiver", + "verification": [], "instances": [ "@LinkArchiver twitter bot" ] @@ -3363,6 +3733,7 @@ "pattern": "YaK\\/", "addition_date": "2017/09/25", "url": "http://linkfluence.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" ] @@ -3375,6 +3746,7 @@ "pattern": "PaperLiBot", "addition_date": "2017/09/25", "url": "http://support.paper.li/entries/20023257-what-is-paper-li", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)", "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)" @@ -3388,6 +3760,7 @@ "pattern": "Digg Deeper", "addition_date": "2017/09/26", "url": "http://digg.com/about", + "verification": [], "instances": [ "Digg Deeper/v1 (http://digg.com/about)" ] @@ -3400,6 +3773,7 @@ "pattern": "dcrawl", "addition_date": "2017/09/22", "url": "https://github.com/kgretzky/dcrawl", + "verification": [], "instances": [ "dcrawl/1.0" ] @@ -3412,6 +3786,7 @@ "pattern": "Snacktory", "addition_date": "2017/09/23", "url": "https://github.com/karussell/snacktory", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" ] @@ -3424,6 +3799,7 @@ "pattern": "AndersPinkBot", "addition_date": "2017/09/24", "url": "http://anderspink.com/bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)" ] @@ -3435,6 +3811,7 @@ ], "pattern": "Fyrebot", "addition_date": "2017/09/22", + "verification": [], "instances": [ "Fyrebot/1.0" ] @@ -3447,6 +3824,7 @@ "pattern": "EveryoneSocialBot", "addition_date": "2017/09/22", "url": "http://everyonesocial.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)" ] @@ -3459,6 +3837,7 @@ "pattern": "Mediatoolkitbot", "addition_date": "2017/10/06", "url": "http://mediatoolkit.com", + "verification": [], "instances": [ "Mediatoolkitbot (complaints@mediatoolkit.com)" ] @@ -3470,6 +3849,7 @@ ], "pattern": "Luminator-robots", "addition_date": "2017/09/22", + "verification": [], "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0" ] @@ -3482,6 +3862,7 @@ "pattern": "ExtLinksBot", "addition_date": "2017/11/02", "url": "https://extlinks.com/Bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)" ] @@ -3493,6 +3874,7 @@ ], "pattern": "SurveyBot", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)" ] @@ -3504,6 +3886,7 @@ ], "pattern": "NING\\/", "addition_date": "2017/11/02", + "verification": [], "instances": [ "NING/1.0" ] @@ -3515,6 +3898,7 @@ ], "pattern": "okhttp", "addition_date": "2017/11/02", + "verification": [], "instances": [ "okhttp/2.5.0", "okhttp/2.7.5", @@ -3530,6 +3914,7 @@ ], "pattern": "Nuzzel", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Nuzzel" ] @@ -3542,6 +3927,7 @@ "pattern": "omgili", "addition_date": "2017/11/02", "url": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/", + "verification": [], "instances": [ "omgili/0.5 +http://omgili.com" ] @@ -3554,6 +3940,7 @@ "pattern": "PocketParser", "addition_date": "2017/11/02", "url": "https://getpocket.com/pocketparser_ua", + "verification": [], "instances": [ "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" ] @@ -3565,6 +3952,7 @@ ], "pattern": "YisouSpider", "addition_date": "2017/11/02", + "verification": [], "instances": [ "YisouSpider", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36" @@ -3577,6 +3965,7 @@ ], "pattern": "um-LN", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" ] @@ -3589,6 +3978,7 @@ "pattern": "ToutiaoSpider", "addition_date": "2017/11/02", "url": "http://web.toutiao.com/media_cooperation/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)" ] @@ -3601,6 +3991,7 @@ "pattern": "MuckRack", "addition_date": "2017/11/02", "url": "http://muckrack.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" ] @@ -3613,6 +4004,7 @@ "pattern": "Jamie's Spider", "addition_date": "2017/11/02", "url": "http://jamiembrown.com/", + "verification": [], "instances": [ "Jamie's Spider (http://jamiembrown.com/)" ] @@ -3625,6 +4017,7 @@ "pattern": "AHC\\/", "addition_date": "2017/11/02", "url": "https://github.com/AsyncHttpClient/async-http-client", + "verification": [], "instances": [ "AHC/2.0" ] @@ -3636,6 +4029,7 @@ ], "pattern": "NetcraftSurveyAgent", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)" ] @@ -3647,6 +4041,7 @@ ], "pattern": "Laserlikebot", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)" ] @@ -3658,6 +4053,7 @@ ], "pattern": "^Apache-HttpClient", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Apache-HttpClient/4.2.3 (java 1.5)", "Apache-HttpClient/4.2.5 (java 1.5)", @@ -3683,6 +4079,7 @@ ], "pattern": "AppEngine-Google", "addition_date": "2017/11/02", + "verification": [], "instances": [ "AppEngine-Google; (+http://code.google.com/appengine; appid: example)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)" @@ -3695,6 +4092,7 @@ ], "pattern": "Jetty", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Jetty/9.3.z-SNAPSHOT" ] @@ -3706,6 +4104,7 @@ ], "pattern": "Upflow", "addition_date": "2017/11/02", + "verification": [], "instances": [ "Upflow/1.0" ] @@ -3718,6 +4117,7 @@ "pattern": "Thinklab", "addition_date": "2017/11/02", "url": "https://thinklab.com", + "verification": [], "instances": [ "Thinklab (thinklab.com)" ] @@ -3730,6 +4130,7 @@ "pattern": "Traackr\\.com", "addition_date": "2017/11/02", "url": "https://www.traackr.com/", + "verification": [], "instances": [ "Traackr.com" ] @@ -3742,6 +4143,7 @@ "pattern": "Twurly", "addition_date": "2017/11/02", "url": "http://twurly.org", + "verification": [], "instances": [ "Ruby, Twurly v1.1 (http://twurly.org)" ] @@ -3754,6 +4156,7 @@ ], "pattern": "Mastodon", "addition_date": "2017/11/02", + "verification": [], "instances": [ "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" ] @@ -3765,6 +4168,7 @@ ], "pattern": "http_get", "addition_date": "2017/11/02", + "verification": [], "instances": [ "http_get" ] @@ -3776,6 +4180,7 @@ ], "pattern": "DnyzBot", "addition_date": "2017/11/20", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; DnyzBot/1.0)" ] @@ -3787,6 +4192,7 @@ ], "pattern": "botify", "addition_date": "2018/02/01", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; botify; http://botify.com)" ] @@ -3798,6 +4204,7 @@ ], "pattern": "007ac9 Crawler", "addition_date": "2018/02/09", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)" ] @@ -3809,6 +4216,7 @@ ], "pattern": "BehloolBot", "addition_date": "2018/02/09", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)" ] @@ -3820,6 +4228,7 @@ ], "pattern": "BrandVerity", "addition_date": "2018/02/27", + "verification": [], "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" @@ -3833,6 +4242,7 @@ ], "pattern": "check_http", "addition_date": "2018/02/09", + "verification": [], "instances": [ "check_http/v2.2.1 (nagios-plugins 2.2.1)" ] @@ -3844,6 +4254,7 @@ ], "pattern": "BDCbot", "addition_date": "2018/02/09", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" @@ -3856,6 +4267,7 @@ ], "pattern": "ZumBot", "addition_date": "2018/02/09", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)" ] @@ -3868,6 +4280,7 @@ ], "pattern": "EZID", "addition_date": "2018/02/09", + "verification": [], "instances": [ "EZID (EZID link checker; https://ezid.cdlib.org/)" ] @@ -3880,6 +4293,7 @@ ], "pattern": "ICC-Crawler", "addition_date": "2018/02/28", + "verification": [], "instances": [ "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)" ], @@ -3893,6 +4307,7 @@ ], "pattern": "ArchiveBot", "addition_date": "2018/02/28", + "verification": [], "instances": [ "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)" ], @@ -3905,6 +4320,7 @@ ], "pattern": "^LCC ", "addition_date": "2018/02/28", + "verification": [], "instances": [ "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" ], @@ -3917,6 +4333,7 @@ ], "pattern": "filterdb\\.iss\\.net\\/crawler", "addition_date": "2018/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)" ], @@ -3929,6 +4346,7 @@ ], "pattern": "BLP_bbot", "addition_date": "2018/03/27", + "verification": [], "instances": [ "BLP_bbot/0.1" ] @@ -3940,6 +4358,7 @@ ], "pattern": "BomboraBot", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" ], @@ -3952,6 +4371,7 @@ ], "pattern": "Buck\\/", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" ], @@ -3964,6 +4384,7 @@ ], "pattern": "Companybook-Crawler", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Companybook-Crawler (+https://www.companybooknetworking.com/)" ], @@ -3976,6 +4397,7 @@ ], "pattern": "Genieo", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" ], @@ -3988,6 +4410,7 @@ ], "pattern": "magpie-crawler", "addition_date": "2018/03/27", + "verification": [], "instances": [ "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)" ], @@ -4000,6 +4423,7 @@ ], "pattern": "MeltwaterNews", "addition_date": "2018/03/27", + "verification": [], "instances": [ "MeltwaterNews www.meltwater.com" ], @@ -4012,6 +4436,7 @@ ], "pattern": "Moreover", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" ], @@ -4024,6 +4449,7 @@ ], "pattern": "newspaper\\/", "addition_date": "2018/03/27", + "verification": [], "instances": [ "newspaper/0.1.0.7", "newspaper/0.2.5", @@ -4038,6 +4464,7 @@ ], "pattern": "ScoutJet", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" ], @@ -4050,6 +4477,7 @@ ], "pattern": "(^| )sentry\\/", "addition_date": "2018/03/27", + "verification": [], "instances": [ "sentry/8.22.0 (https://sentry.io)" ], @@ -4062,6 +4490,7 @@ ], "pattern": "StorygizeBot", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)" ], @@ -4074,6 +4503,7 @@ ], "pattern": "UptimeRobot", "addition_date": "2018/03/27", + "verification": [], "instances": [ "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" ], @@ -4086,6 +4516,7 @@ ], "pattern": "OutclicksBot", "addition_date": "2018/04/21", + "verification": [], "instances": [ "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", @@ -4101,6 +4532,7 @@ ], "pattern": "seoscanners", "addition_date": "2018/05/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" ], @@ -4113,6 +4545,7 @@ ], "pattern": "Hatena", "addition_date": "2018/05/29", + "verification": [], "instances": [ "Hatena Antenna/0.3", "Hatena::Russia::Crawler/0.01", @@ -4131,6 +4564,7 @@ ], "pattern": "Google Web Preview", "addition_date": "2018/05/31", + "verification": [], "instances": [ "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36" @@ -4143,6 +4577,7 @@ ], "pattern": "MauiBot", "addition_date": "2018/06/06", + "verification": [], "instances": [ "MauiBot (crawler.feedback+wc@gmail.com)" ] @@ -4154,6 +4589,7 @@ ], "pattern": "AlphaBot", "addition_date": "2018/05/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)" ], @@ -4166,6 +4602,7 @@ ], "pattern": "SBL-BOT", "addition_date": "2018/06/06", + "verification": [], "instances": [ "SBL-BOT (http://sbl.net)" ], @@ -4179,6 +4616,7 @@ ], "pattern": "IAS crawler", "addition_date": "2018/06/06", + "verification": [], "instances": [ "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" ], @@ -4192,6 +4630,7 @@ ], "pattern": "adscanner", "addition_date": "2018/06/24", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; adscanner/)" ] @@ -4203,6 +4642,7 @@ ], "pattern": "Netvibes", "addition_date": "2018/06/24", + "verification": [], "instances": [ "Netvibes (crawler/bot; http://www.netvibes.com", "Netvibes (crawler; http://www.netvibes.com)" @@ -4216,6 +4656,7 @@ ], "pattern": "acapbot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)", "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)" @@ -4228,6 +4669,7 @@ ], "pattern": "Baidu-YunGuanCe", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Baidu-YunGuanCe-Bot(ce.baidu.com)", "Baidu-YunGuanCe-SLABot(ce.baidu.com)", @@ -4245,6 +4687,7 @@ ], "pattern": "bitlybot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "bitlybot/3.0 (+http://bit.ly/)", "bitlybot/2.0", @@ -4259,6 +4702,7 @@ ], "pattern": "blogmuraBot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "blogmuraBot (+http://www.blogmura.com)" ], @@ -4272,6 +4716,7 @@ ], "pattern": "Bot\\.AraTurka\\.com", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Bot.AraTurka.com/0.0.1" ], @@ -4284,6 +4729,7 @@ ], "pattern": "bot-pge\\.chlooe\\.com", "addition_date": "2018/06/27", + "verification": [], "instances": [ "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)" ] @@ -4295,6 +4741,7 @@ ], "pattern": "BoxcarBot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)" ], @@ -4307,6 +4754,7 @@ ], "pattern": "BTWebClient", "addition_date": "2018/06/27", + "verification": [], "instances": [ "BTWebClient/180B(9704)" ], @@ -4320,6 +4768,7 @@ ], "pattern": "ContextAd Bot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", "ContextAd Bot 1.0" @@ -4332,6 +4781,7 @@ ], "pattern": "Digincore bot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)" ], @@ -4344,6 +4794,7 @@ ], "pattern": "Disqus", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Disqus/1.0" ], @@ -4357,6 +4808,7 @@ ], "pattern": "Feedly", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", "FeedlyBot/1.0 (http://feedly.com)" @@ -4371,6 +4823,7 @@ ], "pattern": "Fetch\\/", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" ] @@ -4382,6 +4835,7 @@ ], "pattern": "Fever", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" ], @@ -4394,6 +4848,7 @@ ], "pattern": "Flamingo_SearchEngine", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" ] @@ -4406,6 +4861,7 @@ ], "pattern": "FlipboardProxy", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", @@ -4423,6 +4879,7 @@ ], "pattern": "g2reader-bot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "g2reader-bot/1.0 (+http://www.g2reader.com/)" ], @@ -4435,6 +4892,7 @@ ], "pattern": "G2 Web Services", "addition_date": "2019/03/01", + "verification": [], "instances": [ "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)" ], @@ -4447,6 +4905,7 @@ ], "pattern": "imrbot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)" ], @@ -4459,6 +4918,7 @@ ], "pattern": "K7MLWCBot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "K7MLWCBot/1.0 (+http://www.k7computing.com)" ], @@ -4472,6 +4932,7 @@ ], "pattern": "Kemvibot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)" ], @@ -4484,6 +4945,7 @@ ], "pattern": "Landau-Media-Spider", "addition_date": "2018/06/27", + "verification": [], "instances": [ "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)" ], @@ -4496,6 +4958,7 @@ ], "pattern": "linkapediabot", "addition_date": "2018/06/27", + "verification": [], "instances": [ "linkapediabot (+http://www.linkapedia.com)" ], @@ -4508,6 +4971,7 @@ ], "pattern": "vkShare", "addition_date": "2018/07/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" ], @@ -4520,6 +4984,7 @@ ], "pattern": "Siteimprove\\.com", "addition_date": "2018/06/22", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", @@ -4534,6 +4999,7 @@ ], "pattern": "BLEXBot\\/", "addition_date": "2018/07/07", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" ], @@ -4546,6 +5012,7 @@ ], "pattern": "DareBoost", "addition_date": "2018/07/07", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" ], @@ -4559,6 +5026,7 @@ ], "pattern": "ZuperlistBot\\/", "addition_date": "2018/07/07", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ZuperlistBot/1.0)" ] @@ -4570,6 +5038,7 @@ ], "pattern": "Miniflux\\/", "addition_date": "2018/07/07", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", @@ -4591,6 +5060,7 @@ ], "pattern": "Feedspot", "addition_date": "2018/07/07", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" @@ -4604,6 +5074,7 @@ ], "pattern": "Diffbot\\/", "addition_date": "2018/07/07", + "verification": [], "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)" ], @@ -4616,6 +5087,7 @@ ], "pattern": "SEOkicks", "addition_date": "2018/08/22", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" ], @@ -4628,6 +5100,7 @@ ], "pattern": "tracemyfile", "addition_date": "2018/08/23", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" ] @@ -4639,6 +5112,7 @@ ], "pattern": "Nimbostratus-Bot", "addition_date": "2018/08/29", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" ] @@ -4650,6 +5124,7 @@ ], "pattern": "zgrab", "addition_date": "2018/08/30", + "verification": [], "instances": [ "Mozilla/5.0 zgrab/0.x" ], @@ -4662,6 +5137,7 @@ ], "pattern": "PR-CY\\.RU", "addition_date": "2018/08/30", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" ], @@ -4674,6 +5150,7 @@ ], "pattern": "AdsTxtCrawler", "addition_date": "2018/08/30", + "verification": [], "instances": [ "AdsTxtCrawler/1.0" ] @@ -4685,6 +5162,7 @@ ], "pattern": "Datafeedwatch", "addition_date": "2018/09/05", + "verification": [], "instances": [ "Datafeedwatch/2.1.x" ], @@ -4697,6 +5175,7 @@ ], "pattern": "Zabbix", "addition_date": "2018/09/05", + "verification": [], "instances": [ "Zabbix" ], @@ -4709,6 +5188,7 @@ ], "pattern": "TangibleeBot", "addition_date": "2018/09/05", + "verification": [], "instances": [ "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)" ], @@ -4721,6 +5201,7 @@ ], "pattern": "google-xrawler", "addition_date": "2018/09/05", + "verification": [], "instances": [ "google-xrawler" ], @@ -4733,6 +5214,7 @@ ], "pattern": "axios", "addition_date": "2018/09/06", + "verification": [], "instances": [ "axios/0.18.0", "axios/0.19.0" @@ -4746,6 +5228,7 @@ ], "pattern": "Amazon CloudFront", "addition_date": "2018/09/07", + "verification": [], "instances": [ "Amazon CloudFront" ], @@ -4758,6 +5241,7 @@ ], "pattern": "Pulsepoint", "addition_date": "2018/09/24", + "verification": [], "instances": [ "Pulsepoint XT3 web scraper" ] @@ -4769,6 +5253,7 @@ ], "pattern": "CloudFlare-AlwaysOnline", "addition_date": "2018/09/27", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" @@ -4784,6 +5269,7 @@ ], "pattern": "Google-Structured-Data-Testing-Tool", "addition_date": "2018/10/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)", "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)" @@ -4797,6 +5283,7 @@ ], "pattern": "WordupInfoSearch", "addition_date": "2018/10/07", + "verification": [], "instances": [ "WordupInfoSearch/1.0" ] @@ -4808,6 +5295,7 @@ ], "pattern": "WebDataStats", "addition_date": "2018/10/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" ], @@ -4820,6 +5308,7 @@ ], "pattern": "HttpUrlConnection", "addition_date": "2018/10/08", + "verification": [], "instances": [ "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" ] @@ -4831,6 +5320,7 @@ ], "pattern": "ZoomBot", "addition_date": "2018/10/10", + "verification": [], "instances": [ "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)" ], @@ -4844,6 +5334,7 @@ "pattern": "VelenPublicWebCrawler", "addition_date": "2018/10/09", "url": "https://velen.io/", + "verification": [], "instances": [ "VelenPublicWebCrawler (velen.io)" ] @@ -4855,6 +5346,7 @@ ], "pattern": "MoodleBot", "addition_date": "2018/10/10", + "verification": [], "instances": [ "MoodleBot/1.0" ] @@ -4866,6 +5358,7 @@ ], "pattern": "jpg-newsbot", "addition_date": "2018/10/10", + "verification": [], "instances": [ "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)" ], @@ -4878,6 +5371,7 @@ ], "pattern": "outbrain", "addition_date": "2018/10/14", + "verification": [], "instances": [ "Mozilla/5.0 (Java) outbrain" ], @@ -4890,6 +5384,7 @@ ], "pattern": "W3C_Validator", "addition_date": "2018/10/14", + "verification": [], "instances": [ "W3C_Validator/1.3" ], @@ -4902,6 +5397,7 @@ ], "pattern": "Validator\\.nu", "addition_date": "2018/10/14", + "verification": [], "instances": [ "Validator.nu/LV" ], @@ -4917,6 +5413,7 @@ "depends_on": [ "libwww-perl" ], + "verification": [], "instances": [ "W3C-checklink/2.90 libwww-perl/5.64", "W3C-checklink/3.6.2.3 libwww-perl/5.64", @@ -4937,6 +5434,7 @@ ], "pattern": "W3C-mobileOK", "addition_date": "2018/10/14", + "verification": [], "instances": [ "W3C-mobileOK/DDC-1.0" ], @@ -4949,6 +5447,7 @@ ], "pattern": "W3C_I18n-Checker", "addition_date": "2018/10/14", + "verification": [], "instances": [ "W3C_I18n-Checker/1.0" ], @@ -4961,6 +5460,7 @@ ], "pattern": "FeedValidator", "addition_date": "2018/10/14", + "verification": [], "instances": [ "FeedValidator/1.3" ], @@ -4973,6 +5473,7 @@ ], "pattern": "W3C_CSS_Validator", "addition_date": "2018/10/14", + "verification": [], "instances": [ "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0" ], @@ -4985,6 +5486,7 @@ ], "pattern": "W3C_Unicorn", "addition_date": "2018/10/14", + "verification": [], "instances": [ "W3C_Unicorn/1.0" ], @@ -4997,6 +5499,7 @@ ], "pattern": "Google-PhysicalWeb", "addition_date": "2018/10/21", + "verification": [], "instances": [ "Mozilla/5.0 (Google-PhysicalWeb)" ] @@ -5008,6 +5511,7 @@ ], "pattern": "Blackboard", "addition_date": "2018/10/28", + "verification": [], "instances": [ "Blackboard Safeassign" ], @@ -5020,6 +5524,7 @@ ], "pattern": "ICBot\\/", "addition_date": "2018/10/23", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz" ], @@ -5032,6 +5537,7 @@ ], "pattern": "BazQux", "addition_date": "2018/10/23", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" ], @@ -5044,6 +5550,7 @@ ], "pattern": "Twingly", "addition_date": "2018/10/23", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" ], @@ -5056,6 +5563,7 @@ ], "pattern": "Rivva", "addition_date": "2018/10/23", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" ], @@ -5068,6 +5576,7 @@ ], "pattern": "Experibot", "addition_date": "2018/11/03", + "verification": [], "instances": [ "Experibot-v2 http://goo.gl/ZAr8wX", "Experibot-v3 http://goo.gl/ZAr8wX" @@ -5081,6 +5590,7 @@ ], "pattern": "awesomecrawler", "addition_date": "2018/11/24", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler" ] @@ -5092,6 +5602,7 @@ ], "pattern": "Dataprovider\\.com", "addition_date": "2018/11/24", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Dataprovider.com)" ], @@ -5104,6 +5615,7 @@ ], "pattern": "GroupHigh\\/", "addition_date": "2018/11/24", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" ], @@ -5116,6 +5628,7 @@ ], "pattern": "theoldreader\\.com", "addition_date": "2018/12/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; theoldreader.com)" ], @@ -5128,6 +5641,7 @@ ], "pattern": "AnyEvent", "addition_date": "2018/12/07", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" ], @@ -5140,6 +5654,7 @@ ], "pattern": "Uptimebot\\.org", "addition_date": "2019/01/17", + "verification": [], "instances": [ "Uptimebot.org - Free website monitoring" ], @@ -5152,6 +5667,7 @@ ], "pattern": "Nmap Scripting Engine", "addition_date": "2019/02/04", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" ], @@ -5164,6 +5680,7 @@ ], "pattern": "2ip\\.ru", "addition_date": "2019/02/12", + "verification": [], "instances": [ "2ip.ru CMS Detector (https://2ip.ru/cms/)" ], @@ -5176,6 +5693,7 @@ ], "pattern": "Clickagy", "addition_date": "2019/02/19", + "verification": [], "instances": [ "Clickagy Intelligence Bot v2" ], @@ -5188,6 +5706,7 @@ ], "pattern": "Caliperbot", "addition_date": "2019/03/02", + "verification": [], "instances": [ "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)" ], @@ -5200,6 +5719,7 @@ ], "pattern": "MBCrawler", "addition_date": "2019/03/02", + "verification": [], "instances": [ "MBCrawler/1.0 (https://monitorbacklinks.com)" ], @@ -5212,6 +5732,7 @@ ], "pattern": "online-webceo-bot", "addition_date": "2019/03/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)" ], @@ -5224,6 +5745,7 @@ ], "pattern": "B2B Bot", "addition_date": "2019/03/02", + "verification": [], "instances": [ "B2B Bot" ] @@ -5235,6 +5757,7 @@ ], "pattern": "AddSearchBot", "addition_date": "2019/03/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)" ], @@ -5248,6 +5771,7 @@ ], "pattern": "Google Favicon", "addition_date": "2019/03/14", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" ] @@ -5259,6 +5783,7 @@ ], "pattern": "HubSpot", "addition_date": "2019/04/15", + "verification": [], "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com", "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", @@ -5274,6 +5799,7 @@ ], "pattern": "Chrome-Lighthouse", "addition_date": "2019/03/15", + "verification": [], "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse", @@ -5290,6 +5816,7 @@ "pattern": "HeadlessChrome", "url": "https://developers.google.com/web/updates/2017/04/headless-chrome", "addition_date": "2019/06/17", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36", @@ -5303,6 +5830,7 @@ ], "pattern": "CheckMarkNetwork\\/", "addition_date": "2019/06/30", + "verification": [], "instances": [ "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" ], @@ -5315,6 +5843,7 @@ ], "pattern": "www\\.uptime\\.com", "addition_date": "2019/07/21", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)" ], @@ -5327,6 +5856,7 @@ ], "pattern": "Streamline3Bot\\/", "addition_date": "2019/07/21", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0" @@ -5340,6 +5870,7 @@ ], "pattern": "serpstatbot\\/", "addition_date": "2019/07/25", + "verification": [], "instances": [ "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)" @@ -5353,6 +5884,7 @@ ], "pattern": "MixnodeCache\\/", "addition_date": "2019/08/04", + "verification": [], "instances": [ "MixnodeCache/1.8(+https://cache.mixnode.com/)" ], @@ -5365,6 +5897,7 @@ ], "pattern": "^curl", "addition_date": "2019/08/15", + "verification": [], "instances": [ "curl", "curl/7.29.0", @@ -5384,6 +5917,7 @@ ], "pattern": "SimpleScraper", "addition_date": "2019/08/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SimpleScraper)" ], @@ -5396,6 +5930,7 @@ ], "pattern": "RSSingBot", "addition_date": "2019/09/15", + "verification": [], "instances": [ "RSSingBot (http://www.rssing.com)" ], @@ -5408,6 +5943,7 @@ ], "pattern": "Jooblebot", "addition_date": "2019/09/25", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36" ], @@ -5420,6 +5956,7 @@ ], "pattern": "fedoraplanet", "addition_date": "2019/09/28", + "verification": [], "instances": [ "venus/fedoraplanet" ], @@ -5432,6 +5969,7 @@ ], "pattern": "Friendica", "addition_date": "2019/09/28", + "verification": [], "instances": [ "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz" ], @@ -5444,6 +5982,7 @@ ], "pattern": "NextCloud", "addition_date": "2019/09/30", + "verification": [], "instances": [ "NextCloud-News/1.0" ], @@ -5456,6 +5995,7 @@ ], "pattern": "Tiny Tiny RSS", "addition_date": "2019/10/04", + "verification": [], "instances": [ "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)", "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)", @@ -5471,6 +6011,7 @@ ], "pattern": "RegionStuttgartBot", "addition_date": "2019/10/17", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)" ], @@ -5483,6 +6024,7 @@ ], "pattern": "Bytespider", "addition_date": "2019/11/11", + "verification": [], "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider", @@ -5513,6 +6055,7 @@ ], "pattern": "Datanyze", "addition_date": "2019/11/17", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" ], @@ -5526,6 +6069,15 @@ ], "pattern": "Google-Site-Verification", "addition_date": "2019/12/11", + "verification": [ + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], "instances": [ "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)" ], @@ -5539,6 +6091,7 @@ ], "pattern": "TrendsmapResolver", "addition_date": "2020/02/24", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)" ], @@ -5551,6 +6104,7 @@ ], "pattern": "tweetedtimes", "addition_date": "2020/02/24", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; +http://tweetedtimes.com)" ], @@ -5563,6 +6117,7 @@ ], "pattern": "NTENTbot", "addition_date": "2020/02/24", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; NTENTbot; +http://www.ntent.com/ntentbot)" ], @@ -5575,6 +6130,7 @@ ], "pattern": "Gwene", "addition_date": "2020/02/24", + "verification": [], "instances": [ "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot" ], @@ -5587,6 +6143,7 @@ ], "pattern": "SimplePie", "addition_date": "2020/02/24", + "verification": [], "instances": [ "SimplePie/1.3-dev (Feed Parser; http://simplepie.org; Allow like Gecko)" ], @@ -5599,6 +6156,7 @@ ], "pattern": "SearchAtlas", "addition_date": "2020/03/02", + "verification": [], "instances": [ "SearchAtlas.com SEO Crawler" ], @@ -5611,6 +6169,7 @@ ], "pattern": "Superfeedr", "addition_date": "2020/03/02", + "verification": [], "instances": [ "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860" ], @@ -5623,6 +6182,7 @@ ], "pattern": "feedbot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "wp.com feedbot/1.0 (+https://wp.com)" ], @@ -5635,6 +6195,7 @@ ], "pattern": "UT-Dorkbot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "UT-Dorkbot/1.0" ], @@ -5647,6 +6208,7 @@ ], "pattern": "Amazonbot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" ], @@ -5659,6 +6221,7 @@ ], "pattern": "SerendeputyBot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)" ], @@ -5671,6 +6234,7 @@ ], "pattern": "Eyeotabot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Eyeotabot/1.0; +http://www.eyeota.com)" ], @@ -5684,6 +6248,7 @@ ], "pattern": "officestorebot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; officestorebot/1.0; +https://aka.ms/officestorebot)" ], @@ -5696,6 +6261,7 @@ ], "pattern": "Neticle Crawler", "addition_date": "2020/03/02", + "verification": [], "instances": [ "Neticle Crawler v1.0 ( https://neticle.com/bot/en/ )" ], @@ -5708,6 +6274,7 @@ ], "pattern": "SurdotlyBot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)" ], @@ -5720,6 +6287,7 @@ ], "pattern": "LinkisBot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" ] @@ -5731,6 +6299,7 @@ ], "pattern": "AwarioSmartBot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" ], @@ -5743,6 +6312,7 @@ ], "pattern": "AwarioRssBot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" ], @@ -5755,6 +6325,7 @@ ], "pattern": "RyteBot", "addition_date": "2020/03/02", + "verification": [], "instances": [ "RyteBot/1.0.0 (+https://bot.ryte.com/)" ], @@ -5767,6 +6338,7 @@ ], "pattern": "FreeWebMonitoring SiteChecker", "addition_date": "2020/03/02", + "verification": [], "instances": [ "FreeWebMonitoring SiteChecker/0.2 (+https://www.freewebmonitoring.com/bot.html)" ], @@ -5779,6 +6351,7 @@ ], "pattern": "AspiegelBot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)" ], @@ -5791,6 +6364,7 @@ ], "pattern": "NAVER Blog Rssbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "NAVER Blog Rssbot" ], @@ -5803,6 +6377,7 @@ ], "pattern": "zenback bot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; zenback bot; powered by logly +http://corp.logly.co.jp/)" ], @@ -5815,6 +6390,7 @@ ], "pattern": "SentiBot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "SentiBot www.sentibot.eu (compatible with Googlebot)" ], @@ -5828,6 +6404,7 @@ ], "pattern": "Domains Project\\/", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Domains Project/1.0.3; +https://github.com/tb0hdan/domains)" ], @@ -5840,6 +6417,7 @@ ], "pattern": "Pandalytics", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Pandalytics/1.0 (https://domainsbot.com/pandalytics/)" ], @@ -5852,6 +6430,7 @@ ], "pattern": "VKRobot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; VKRobot/1.0)" ] @@ -5863,6 +6442,7 @@ ], "pattern": "bidswitchbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "bidswitchbot/1.0" ], @@ -5875,6 +6455,7 @@ ], "pattern": "tigerbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "tigerbot" ] @@ -5886,6 +6467,7 @@ ], "pattern": "NIXStatsbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; NIXStatsbot/1.1; +http://www.nixstats.com/bot.html)" ], @@ -5898,6 +6480,7 @@ ], "pattern": "Atom Feed Robot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "RSSMicro.com RSS/Atom Feed Robot" ], @@ -5910,6 +6493,7 @@ ], "pattern": "[Cc]urebot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Curebot/1.0", "curebot-feed-fetcher" @@ -5923,6 +6507,7 @@ ], "pattern": "PagePeeker\\/", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 (compatible; PagePeeker/3.0; +https://pagepeeker.com/robots/)" ], @@ -5935,6 +6520,7 @@ ], "pattern": "Vigil\\/", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)" ], @@ -5947,6 +6533,7 @@ ], "pattern": "rssbot\\/", "addition_date": "2020/03/16", + "verification": [], "instances": [ "rssbot/1.4.3 (+https://t.me/RustRssBot)" ], @@ -5959,6 +6546,7 @@ ], "pattern": "startmebot\\/", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)" ], @@ -5971,6 +6559,7 @@ ], "pattern": "JobboerseBot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0" ], @@ -5983,6 +6572,7 @@ ], "pattern": "seewithkids", "addition_date": "2020/03/16", + "verification": [], "instances": [ "http://seewithkids.com/bot" ], @@ -5995,6 +6585,7 @@ ], "pattern": "NINJA bot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "NINJA bot" ] @@ -6006,6 +6597,7 @@ ], "pattern": "Cutbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Cutbot; 1.5; http://cutbot.net/" ], @@ -6018,6 +6610,7 @@ ], "pattern": "BublupBot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "BublupBot (+https://www.bublup.com/bublup-bot.html)" ], @@ -6030,6 +6623,7 @@ ], "pattern": "BrandONbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "BrandONbot (http://brandonmedia.net)" ], @@ -6042,6 +6636,7 @@ ], "pattern": "RidderBot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)", "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" @@ -6055,6 +6650,7 @@ ], "pattern": "Taboolabot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com)" ], @@ -6068,6 +6664,7 @@ ], "pattern": "Dubbotbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Dubbotbot/0.2; +http://dubbot.com)" ], @@ -6080,6 +6677,7 @@ ], "pattern": "FindITAnswersbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)" ], @@ -6092,6 +6690,7 @@ ], "pattern": "infoobot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "infoobot/0.1 (https://www.infoo.nl/bot.html)" ], @@ -6104,6 +6703,7 @@ ], "pattern": "Refindbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 (Refindbot/1.0)" ], @@ -6116,6 +6716,7 @@ ], "pattern": "BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)" ], @@ -6128,6 +6729,7 @@ ], "pattern": "SeobilityBot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)" ], @@ -6140,6 +6742,7 @@ ], "pattern": "Cincraw", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" ], @@ -6152,6 +6755,7 @@ ], "pattern": "Dragonbot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0; Dragonbot; http://www.dragonmetrics.com" ], @@ -6164,6 +6768,7 @@ ], "pattern": "VoluumDSP-content-bot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; VoluumDSP-content-bot/2.0; +dsp-dev@codewise.com)" ], @@ -6176,6 +6781,7 @@ ], "pattern": "FreshRSS", "addition_date": "2020/03/16", + "verification": [], "instances": [ "FreshRSS/1.11.2 (Linux; https://freshrss.org) like Googlebot" ], @@ -6188,6 +6794,7 @@ ], "pattern": "BitBot", "addition_date": "2020/03/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BitBot/v1.19.0; +https://bitbot.dev)" ], @@ -6200,6 +6807,7 @@ ], "pattern": "^PHP-Curl-Class", "addition_date": "2020/12/10", + "verification": [], "instances": [ "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.2.24 curl/7.61.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.19 curl/7.66.0", @@ -6219,6 +6827,7 @@ ], "pattern": "Google-Certificates-Bridge", "addition_date": "2020/12/23", + "verification": [], "instances": [ "Google-Certificates-Bridge" ] @@ -6230,6 +6839,7 @@ ], "pattern": "centurybot", "addition_date": "2022/04/26", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)" ] @@ -6242,6 +6852,7 @@ ], "pattern": "Viber", "addition_date": "2021/04/27", + "verification": [], "instances": [ "Viber" ], @@ -6255,6 +6866,7 @@ "pattern": "e\\.ventures Investment Crawler", "addition_date": "2021/06/05", "url": "https://www.eventures.vc/", + "verification": [], "instances": [ "e.ventures Investment Crawler (eventures.vc)" ] @@ -6267,6 +6879,7 @@ "pattern": "evc-batch", "addition_date": "2021/06/07", "url": "https://www.eventures.vc/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; evc-batch/2.0)" ] @@ -6279,6 +6892,7 @@ ], "pattern": "PetalBot", "addition_date": "2021/06/07", + "verification": [], "instances": [ "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" @@ -6292,6 +6906,7 @@ ], "pattern": "virustotal", "addition_date": "2021/09/22", + "verification": [], "instances": [ "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)", "AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)" @@ -6306,6 +6921,7 @@ ], "pattern": "(^| )PTST\\/", "addition_date": "2021/12/05", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 PTST/211202.211915", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0 PTST/211202.211915" @@ -6320,6 +6936,7 @@ ], "pattern": "minicrawler", "addition_date": "2022/01/12", + "verification": [], "instances": [ "Testomatobot/1.0 (Linux x86_64; +https://www.testomato.com/testomatobot) minicrawler/5.2.2" ], @@ -6333,6 +6950,7 @@ "pattern": "Cookiebot", "addition_date": "2022/01/23", "url": "https://www.cookiebot.com/", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0; +http://cookiebot.com/) Chrome/97.0.4692.71 Safari/537.36" ] @@ -6345,6 +6963,7 @@ "pattern": "trovitBot", "addition_date": "2022/06/08", "url": "http://www.trovit.com/bot.html", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)" ] @@ -6357,6 +6976,7 @@ "pattern": "seostar\\.co", "addition_date": "2022/08/04", "url": "https://seostar.co/robot/", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)" ] @@ -6369,6 +6989,7 @@ "pattern": "IonCrawl", "addition_date": "2022/08/04", "url": "https://www.ionos.de/terms-gtc/faq-crawler-en", + "verification": [], "instances": [ "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)" ] @@ -6381,6 +7002,7 @@ "pattern": "Uptime-Kuma", "addition_date": "2022/10/17", "url": "https://uptime.kuma.pet/", + "verification": [], "instances": [ "Uptime-Kuma/1.18.0" ] @@ -6393,6 +7015,7 @@ "pattern": "Seekport", "addition_date": "2022/10/17", "url": "https://bot.seekport.com", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)", "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)" @@ -6406,6 +7029,7 @@ "pattern": "FreshpingBot", "addition_date": "2022/10/17", "url": "https://www.freshworks.com/website-monitoring/", + "verification": [], "instances": [ "FreshpingBot/1.0 (+https://freshping.io/)" ] @@ -6418,6 +7042,7 @@ "pattern": "Feedbin", "addition_date": "2022/11/05", "url": "https://feedbin.com/", + "verification": [], "instances": [ "Feedbin feed-id:2005098 - 2 subscribers" ] @@ -6430,6 +7055,7 @@ "pattern": "CriteoBot", "addition_date": "2022/11/13", "url": "https://www.criteo.com/", + "verification": [], "instances": [ "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)" ] @@ -6443,6 +7069,7 @@ "pattern": "Snap URL Preview Service", "addition_date": "2022/11/13", "url": "https://developers.snap.com/robots", + "verification": [], "instances": [ "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" ] @@ -6455,6 +7082,7 @@ "pattern": "Better Uptime Bot", "addition_date": "2022/11/13", "url": "https://betteruptime.com/", + "verification": [], "instances": [ "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" ] @@ -6467,6 +7095,7 @@ "pattern": "RuxitSynthetic", "addition_date": "2023/02/16", "url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent", + "verification": [], "instances": [ "RuxitSynthetic/1.0" ] @@ -6479,6 +7108,7 @@ "pattern": "Google-Read-Aloud", "addition_date": "2023/02/16", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)" @@ -6492,6 +7122,7 @@ ], "pattern": "Valve\\/Steam", "addition_date": "2023/05/24", + "verification": [], "instances": [ "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)" ] @@ -6503,6 +7134,7 @@ ], "pattern": "OdklBot\\/", "addition_date": "2023/05/24", + "verification": [], "instances": [ "OdklBot/1.0 (share@odnoklassniki.ru)", "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)" @@ -6516,6 +7148,7 @@ ], "pattern": "GPTBot", "addition_date": "2023/08/09", + "verification": [], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)" ], @@ -6528,6 +7161,7 @@ ], "pattern": "ChatGPT-User", "addition_date": "2024/04/19", + "verification": [], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot" ], @@ -6540,6 +7174,16 @@ ], "pattern": "YandexRenderResourcesBot\\/", "addition_date": "2023/08/16", + "verification": [ + { + "type": "dns", + "masks": [ + "@.yandex.ru", + "@.yandex.com", + "@.yandex.net" + ] + } + ], "instances": [ "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0" ], @@ -6552,6 +7196,7 @@ ], "pattern": "LightspeedSystemsCrawler", "addition_date": "2023/08/16", + "verification": [], "instances": [ "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US" ] @@ -6563,6 +7208,7 @@ ], "pattern": "ev-crawler\\/", "addition_date": "2023/08/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)" ], @@ -6575,6 +7221,7 @@ ], "pattern": "BitSightBot\\/", "addition_date": "2023/08/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; BitSightBot/1.0)" ], @@ -6587,6 +7234,7 @@ ], "pattern": "woorankreview\\/", "addition_date": "2023/08/16", + "verification": [], "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)", "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)" @@ -6600,6 +7248,14 @@ ], "pattern": "Google-Safety", "addition_date": "2023/08/17", + "verification": [ + { + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" + ] + } + ], "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", @@ -6614,6 +7270,7 @@ ], "pattern": "AwarioBot", "addition_date": "2023/08/23", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)" ], @@ -6626,6 +7283,7 @@ ], "pattern": "DataForSeoBot", "addition_date": "2023/08/23", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)" ], @@ -6638,6 +7296,7 @@ ], "pattern": "Linespider", "addition_date": "2023/08/24", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36" @@ -6651,6 +7310,7 @@ ], "pattern": "WellKnownBot", "addition_date": "2023/08/29", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)" ], @@ -6663,6 +7323,7 @@ ], "pattern": "A Patent Crawler", "addition_date": "2023/08/29", + "verification": [], "instances": [ "E. Orliac, G. Fourestey/2.3 (A Patent Crawler; http://scitas.epfl.ch/; etienne.orliac@epfl.ch, gilles.fourestey@epfl.ch)" ], @@ -6675,6 +7336,7 @@ ], "pattern": "StractBot", "addition_date": "2023/09/06", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)" ], @@ -6687,6 +7349,7 @@ ], "pattern": "search\\.marginalia\\.nu", "addition_date": "2023/09/08", + "verification": [], "instances": [ "search.marginalia.nu" ], @@ -6699,6 +7362,7 @@ ], "pattern": "YouBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "YouBot (+http://www.you.com)" ], @@ -6711,6 +7375,7 @@ ], "pattern": "Nicecrawler", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Nicecrawler/1.1; +http://www.nicecrawler.com/) Chrome/90.0.4430.97 Safari/537.36" ], @@ -6723,6 +7388,7 @@ ], "pattern": "Neevabot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)" ], @@ -6735,6 +7401,7 @@ ], "pattern": "BrightEdge Crawler", "addition_date": "2023/09/08", + "verification": [], "instances": [ "BrightEdge Crawler/1.0 (crawler@brightedge.com)" ], @@ -6747,6 +7414,7 @@ ], "pattern": "SiteCheckerBotCrawler", "addition_date": "2023/09/08", + "verification": [], "instances": [ "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)" ], @@ -6759,6 +7427,7 @@ ], "pattern": "TombaPublicWebCrawler", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; TombaPublicWebCrawler/1.0; +https://tombascraper.com)" ], @@ -6771,6 +7440,7 @@ ], "pattern": "CrawlyProjectCrawler", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; crawlyproject@digitaldragon.dev +https://crawlyproject.digitaldragon.dev/)" ], @@ -6783,6 +7453,7 @@ ], "pattern": "KomodiaBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0" ], @@ -6795,6 +7466,7 @@ ], "pattern": "KStandBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://url-classification.io/wiki/index.php?title=URL_server_crawler) KStandBot/1.0" ], @@ -6807,6 +7479,7 @@ ], "pattern": "CISPA Webcrawler", "addition_date": "2023/09/08", + "verification": [], "instances": [ "CISPA Webcrawler (https://vuln-notify-checker.cispa.saarland)" ], @@ -6819,6 +7492,7 @@ ], "pattern": "MTRobot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)" ], @@ -6831,6 +7505,7 @@ ], "pattern": "hyscore\\.io", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; HyScore/1.0; +https://hyscore.io/crawler/)" ], @@ -6843,6 +7518,7 @@ ], "pattern": "AlexandriaOrgBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (Linux) (compatible; AlexandriaOrgBot/1.0; +https://www.alexandria.org/bot.html)" ], @@ -6855,6 +7531,7 @@ ], "pattern": "2ip bot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "2ip bot/1.1 (+http://2ip.io)" ], @@ -6867,6 +7544,7 @@ ], "pattern": "Yellowbrandprotectionbot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Yellowbrandprotectionbot/1.0; +https://www.yellowbp.com/bot.html)" ], @@ -6879,6 +7557,7 @@ ], "pattern": "SEOlizer", "addition_date": "2023/09/08", + "verification": [], "instances": [ "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" ], @@ -6891,6 +7570,7 @@ ], "pattern": "vuhuvBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)" ], @@ -6903,6 +7583,7 @@ ], "pattern": "INETDEX-BOT", "addition_date": "2023/09/08", + "verification": [], "instances": [ "INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/bot.html)" ], @@ -6916,6 +7597,7 @@ ], "pattern": "Synapse", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Synapse (bot; +https://github.com/matrix-org/synapse)" ], @@ -6928,6 +7610,7 @@ ], "pattern": "t3versionsBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; t3versionsBot/1.0; +https://www.t3versions.com/bot)" ], @@ -6940,6 +7623,7 @@ ], "pattern": "deepnoc", "addition_date": "2023/09/08", + "verification": [], "instances": [ "deepnoc - https://deepnoc.com/bot" ], @@ -6952,6 +7636,7 @@ ], "pattern": "Cocolyzebot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)" ], @@ -6964,6 +7649,7 @@ ], "pattern": "hypestat", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; hypestat/1.0; +https://hypestat.com/bot)" ], @@ -6976,6 +7662,7 @@ ], "pattern": "ReverseEngineeringBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ReverseEngineeringBot/0.1; +https://torus.company/bot.html)" ], @@ -6988,6 +7675,7 @@ ], "pattern": "sempi\\.tech", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)" ], @@ -7001,6 +7689,7 @@ ], "pattern": "Iframely", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Iframely/1.3.1 (+https://iframely.com/docs/about) Atlassian" ], @@ -7013,6 +7702,7 @@ ], "pattern": "MetaInspector", "addition_date": "2023/09/08", + "verification": [], "instances": [ "MetaInspector/5.6.0 (+https://github.com/jaimeiniesta/metainspector)" ], @@ -7025,6 +7715,7 @@ ], "pattern": "node-fetch", "addition_date": "2023/09/08", + "verification": [], "instances": [ "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" ], @@ -7037,6 +7728,7 @@ ], "pattern": "lkxscan", "addition_date": "2023/09/08", + "verification": [], "instances": [ "lkxscan/v0.1.0 (+https://leakix.net) l9explore/v1.0.0 (+https://github.com/LeakIX/l9explore)" ], @@ -7049,6 +7741,7 @@ ], "pattern": "python-opengraph", "addition_date": "2023/09/08", + "verification": [], "instances": [ "python-opengraph-jaywink/0.2.0 (+https://github.com/jaywink/python-opengraph)" ], @@ -7061,6 +7754,7 @@ ], "pattern": "OpenGraphCheck", "addition_date": "2023/09/08", + "verification": [], "instances": [ "OpenGraphCheck/2.1 (+https://opengraphcheck.com)" ], @@ -7075,6 +7769,7 @@ ], "pattern": "developers\\.google\\.com\\/\\+\\/web\\/snippet", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/" @@ -7088,6 +7783,7 @@ ], "pattern": "SenutoBot", "addition_date": "2023/09/08", + "verification": [], "instances": [ "SenutoBot/1.0 (compatible; SenutoBot/1.0; +https://www.senuto.com/)" ], @@ -7100,6 +7796,7 @@ ], "pattern": "MaCoCu", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)" ], @@ -7112,6 +7809,7 @@ ], "pattern": "NewsBlur", "addition_date": "2023/09/08", + "verification": [], "instances": [ "NewsBlur Feed Fetcher - 1 subscriber - http://www.newsblur.com/site/0000000/webpage (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15)" ], @@ -7124,6 +7822,7 @@ ], "pattern": "inoreader", "addition_date": "2023/09/08", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; inoreader.com; 1 subscribers)" ], @@ -7136,6 +7835,7 @@ ], "pattern": "NetSystemsResearch", "addition_date": "2023/09/08", + "verification": [], "instances": [ "NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com" ], @@ -7148,6 +7848,7 @@ ], "pattern": "PageThing", "addition_date": "2023/09/08", + "verification": [], "instances": [ "PageThing http://pagething.com curl www" ], @@ -7160,6 +7861,7 @@ ], "pattern": "WordPress\\/", "addition_date": "2023/10/24", + "verification": [], "instances": [ "WordPress/X.X.X; https://example.com" ], @@ -7172,6 +7874,7 @@ ], "pattern": "PhxBot", "addition_date": "2024/01/06", + "verification": [], "instances": [ "PhxBot/0.1 (phxbot@protonmail.com)" ] @@ -7183,6 +7886,7 @@ ], "pattern": "ImagesiftBot", "addition_date": "2024/01/06", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)" ], @@ -7195,6 +7899,7 @@ ], "pattern": "Expanse", "addition_date": "2024/02/01", + "verification": [], "instances": [ "Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers' presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: scaninfo@paloaltonetworks.com" ], @@ -7207,6 +7912,7 @@ ], "pattern": "InternetMeasurement", "addition_date": "2024/02/01", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; InternetMeasurement/1.0; +https://internet-measurement.com/)" ], @@ -7219,6 +7925,7 @@ ], "pattern": "^BW\\/", "addition_date": "2024/02/08", + "verification": [], "instances": [ "BW/1.1; bit.ly/3eZNDnO", "BW/1.1; rb.gy/oupwis" @@ -7232,6 +7939,7 @@ ], "pattern": "GeedoBot", "addition_date": "2024/02/11", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; GeedoBot; +http://www.geedo.com/bot.html)" ], @@ -7244,6 +7952,7 @@ ], "pattern": "Audisto Crawler", "addition_date": "2024/03/14", + "verification": [], "instances": [ "Audisto Crawler (mobile; +https://audisto.com/bot)", "Audisto Crawler (desktop; +https://audisto.com/bot)", @@ -7259,6 +7968,7 @@ ], "pattern": "PerplexityBot\\/", "addition_date": "2024/03/14", + "verification": [], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)" ], @@ -7271,6 +7981,7 @@ ], "pattern": "[cC]laude(?:[bB]ot|-[Ww]eb)", "addition_date": "2024/04/19", + "verification": [], "instances": [ "claudebot", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", @@ -7286,6 +7997,7 @@ ], "pattern": "Monsidobot", "addition_date": "2024/05/14", + "verification": [], "instances": [ "Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)" ], @@ -7298,6 +8010,7 @@ ], "pattern": "GroupMeBot", "addition_date": "2024/05/19", + "verification": [], "instances": [ "GroupMeBot/1.0" ], @@ -7310,6 +8023,7 @@ ], "pattern": "Vercelbot", "addition_date": "2024/08/30", + "verification": [], "instances": [ "Vercelbot (+https://vercel.com)" ], @@ -7324,6 +8038,7 @@ ], "pattern": "vercel-screenshot", "addition_date": "2024/08/30", + "verification": [], "instances": [] }, { @@ -7333,6 +8048,7 @@ ], "pattern": "Coda-Server-Fetcher", "addition_date": "2024/09/10", + "verification": [], "instances": [], "url": "https://coda.io/product/packs" }, @@ -7343,6 +8059,7 @@ ], "pattern": "AI2Bot\\s", "addition_date": "2024/09/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)" ], @@ -7355,6 +8072,7 @@ ], "pattern": "Ai2Bot-Dolma", "addition_date": "2024/09/16", + "verification": [], "instances": [ "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)" ], @@ -7367,6 +8085,7 @@ ], "pattern": "FriendlyCrawler", "addition_date": "2024/09/16", + "verification": [], "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; FriendlyCrawler/1.0) Chrome/120.0.6099.216 Safari/605.1.15" ], @@ -7380,6 +8099,15 @@ ], "pattern": "Google-CloudVertexBot", "addition_date": "2024/09/16", + "verification": [ + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": [ "Google-CloudVertexBot" ], @@ -7393,6 +8121,7 @@ ], "pattern": "[Mm]eta-[Ee]xternal[Aa]gent", "addition_date": "2024/09/16", + "verification": [], "instances": [ "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalagent/1.1" @@ -7408,6 +8137,7 @@ ], "pattern": "meta-externalfetcher", "addition_date": "2024/09/16", + "verification": [], "instances": [ "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalfetcher/1.1" @@ -7422,6 +8152,7 @@ ], "pattern": "OAI-SearchBot", "addition_date": "2024/09/16", + "verification": [], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot" ], @@ -7434,6 +8165,7 @@ ], "pattern": "Timpibot", "addition_date": "2024/09/16", + "verification": [], "instances": [ "Timpibot/0.8 (+http://www.timpi.io)" ], @@ -7446,6 +8178,7 @@ ], "pattern": "webzio\\s", "addition_date": "2024/09/16", + "verification": [], "instances": [ "webzio (+https://webz.io/bot.html)" ], @@ -7458,6 +8191,7 @@ ], "pattern": "webzio-extended", "addition_date": "2024/09/16", + "verification": [], "instances": [ "webzio-extended (+https://webz.io/bot.html)" ], @@ -7470,6 +8204,7 @@ ], "pattern": "cohere-ai", "addition_date": "2024/09/16", + "verification": [], "instances": [] }, { @@ -7480,6 +8215,7 @@ ], "pattern": "iaskspider", "addition_date": "2024/09/16", + "verification": [], "instances": [], "url": "https://neil-clarke.com/block-the-bots-that-feed-ai-models-by-scraping-your-website/" }, @@ -7491,6 +8227,7 @@ ], "pattern": "img2dataset", "addition_date": "2024/09/16", + "verification": [], "instances": [ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 (compatible; img2downloader; +https://github.com/rom1504/img2dataset)" ], @@ -7503,6 +8240,7 @@ ], "pattern": "Datadog\\/{0,1}Synthetics", "addition_date": "2024/09/19", + "verification": [], "instances": [ "Datadog/Synthetics", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 DatadogSynthetics"