diff --git a/Examples/dummy_.htaccess b/Examples/dummy_.htaccess index 01eedb6..db3bc04 100644 --- a/Examples/dummy_.htaccess +++ b/Examples/dummy_.htaccess @@ -7,9 +7,6 @@ ServerSignature Off RewriteEngine On -# Allow e.g. Basic Auth -SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1 - # Store the current location in an environment variable CWD to use # mod_rewrite in .htaccess files without knowing the RewriteBase RewriteCond $0#%{REQUEST_URI} ([^#]*)#(.*)\1$ @@ -38,6 +35,7 @@ RewriteRule .? - [E=TYPO3_CONTEXT:Development/Server] RewriteCond %{HTTP_HOST} ^(www\.)?domain\.localhost$ [NC] RewriteRule (.*) $1 [E=TYPO3_CONTEXT:Development] +# Allow e.g. Basic Auth SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1 ### Example for directorys cache and log outside of document root for better security diff --git a/sites/site_default/robots.yaml b/sites/site_default/robots.yaml index 705de2f..b7a190d 100644 --- a/sites/site_default/robots.yaml +++ b/sites/site_default/robots.yaml @@ -16,1136 +16,1392 @@ routes: route: robots.txt type: staticText # content: end of line change the URL to the sitemap! content: " - User-agent: GPTBot\r\n - Disallow: /\r\n User-agent: *\r\n Disallow: /wp-admin/\r\n Allow: /wp-admin/admin-ajax.php\r\n + User-agent: 01h4x.com\r\n + Disallow:/\r\n User-agent: 360Spider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: 404checker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: 404enemy\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: 80legs\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: ADmantX\r\n + Disallow:/\r\n + User-agent: AIBOT\r\n + Disallow:/\r\n + User-agent: ALittle Client\r\n + Disallow:/\r\n + User-agent: ASPSeek\r\n + Disallow:/\r\n User-agent: Abonti\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Aboundex\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Aboundexbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Acunetix\r\n - Disallow: /\r\n - User-agent: ADmantX\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: AdsTxtCrawlerTP\r\n + Disallow:/\r\n User-agent: AfD-Verbotsverfahren\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: AhrefsBot\r\n - Disallow: /\r\n - User-agent: ia_archiver\r\n - Disallow: /\r\n - User-agent: AIBOT\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: AiHitBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Aipbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Alexibot\r\n - Disallow: /\r\n - User-agent: Alligator\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: AllSubmitter\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Alligator\r\n + Disallow:/\r\n User-agent: AlphaBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Anarchie\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Anarchy\r\n + Disallow:/\r\n + User-agent: Anarchy99\r\n + Disallow:/\r\n + User-agent: Ankit\r\n + Disallow:/\r\n + User-agent: Anthill\r\n + Disallow:/\r\n User-agent: Apexoo\r\n - Disallow: /\r\n - User-agent: archive.org_bot\r\n - Disallow: /\r\n - User-agent: arquivo.pt\r\n - Disallow: /\r\n - User-agent: arquivo-web-crawler\r\n - Disallow: /\r\n - User-agent: ASPSeek\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Aspiegel\r\n + Disallow:/\r\n User-agent: Asterias\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Atomseobot\r\n + Disallow:/\r\n User-agent: Attach\r\n - Disallow: /\r\n - User-agent: autoemailspider\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: AwarioBot\r\n + Disallow:/\r\n User-agent: AwarioRssBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: AwarioSmartBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: BBBike\r\n + Disallow:/\r\n + User-agent: BDCbot\r\n + Disallow:/\r\n + User-agent: BDFetch\r\n + Disallow:/\r\n + User-agent: BLEXBot\r\n + Disallow:/\r\n User-agent: BackDoorBot\r\n - Disallow: /\r\n - User-agent: Backlink-Ceck\r\n - Disallow: /\r\n - User-agent: backlink-check\r\n - Disallow: /\r\n - User-agent: BacklinkCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BackStreet\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BackWeb\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Backlink-Ceck\r\n + Disallow:/\r\n + User-agent: BacklinkCrawler\r\n + Disallow:/\r\n User-agent: Badass\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Bandit\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Barkrowler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BatchFTP\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Battleztar Bazinga\r\n - Disallow: /\r\n - User-agent: BBBike\r\n - Disallow: /\r\n - User-agent: BDCbot\r\n - Disallow: /\r\n - User-agent: BDFetch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BetaBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Bigfoot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Bitacle\r\n - Disallow: /\r\n - User-agent: Blackboard\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Black Hole\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BlackWidow\r\n - Disallow: /\r\n - User-agent: BLEXBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Blackboard\r\n + Disallow:/\r\n User-agent: Blow\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BlowFish\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Boardreader\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Bolt\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BotALot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Brandprotect\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Brandwatch\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Buck\r\n + Disallow:/\r\n User-agent: Buddy\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BuiltBotTough\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BuiltWith\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Bullseye\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BunnySlippers\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: BuzzSumo\r\n - Disallow: /\r\n - User-agent: Calculon\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Bytespider\r\n + Disallow:/\r\n User-agent: CATExplorador\r\n - Disallow: /\r\n - User-agent: CazoodleBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: CCBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: CODE87\r\n + Disallow:/\r\n + User-agent: CSHttp\r\n + Disallow:/\r\n + User-agent: Calculon\r\n + Disallow:/\r\n + User-agent: CazoodleBot\r\n + Disallow:/\r\n User-agent: Cegbfeieh\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: CensysInspect\r\n + Disallow:/\r\n + User-agent: ChatGPT-User\r\n + Disallow:/\r\n + User-agent: CheTeam\r\n + Disallow:/\r\n User-agent: CheeseBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: CherryPicker\r\n - Disallow: /\r\n - User-agent: CheTeam\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: ChinaClaw\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Chlooe\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Citoid\r\n + Disallow:/\r\n User-agent: Claritybot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: ClaudeBot\r\n + Disallow:/\r\n User-agent: Cliqzbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Cloud mapping\r\n - Disallow: /\r\n - User-agent: coccocbot-web\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Cocolyzebot\r\n + Disallow:/\r\n User-agent: Cogentbot\r\n - Disallow: /\r\n - User-agent: cognitiveseo\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Collector\r\n - Disallow: /\r\n - User-agent: com.plumanalytics\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Copier\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: CopyRightCheck\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Copyscape\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Cosmos\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Craftbot\r\n - Disallow: /\r\n - User-agent: crawler4j\r\n - Disallow: /\r\n - User-agent: crawler.feedback\r\n - Disallow: /\r\n - User-agent: crawl.sogou.com\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Crawling at Home Project\r\n + Disallow:/\r\n User-agent: CrazyWebCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Crescent\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: CrunchBot\r\n - Disallow: /\r\n - User-agent: CSHttp\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Curious\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Custo\r\n - Disallow: /\r\n - User-agent: DatabaseDriverMysqli\r\n - Disallow: /\r\n - User-agent: DataCha0s\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: CyotekWebCopy\r\n + Disallow:/\r\n User-agent: DBLBot\r\n - Disallow: /\r\n - User-agent: demandbase-bot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: DIIbot\r\n + Disallow:/\r\n + User-agent: DSearch\r\n + Disallow:/\r\n + User-agent: DTS Agent\r\n + Disallow:/\r\n + User-agent: DataCha0s\r\n + Disallow:/\r\n + User-agent: DatabaseDriverMysqli\r\n + Disallow:/\r\n User-agent: Demon\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Deusu\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Devil\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Digincore\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: DigitalPebble\r\n - Disallow: /\r\n - User-agent: DIIbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Dirbuster\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Disco\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Discobot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Discoverybot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Dispatch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: DittoSpyder\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: DnBCrawler-Analytics\r\n + Disallow:/\r\n User-agent: DnyzBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: DomCopBot\r\n + Disallow:/\r\n User-agent: DomainAppender\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: DomainCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: DomainSigmaCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: DomainStatsBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Domains Project\r\n + Disallow:/\r\n User-agent: Dotbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Download Wonder\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Dragonfly\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Drip\r\n - Disallow: /\r\n - User-agent: DSearch\r\n - Disallow: /\r\n - User-agent: DTS Agent\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: ECCP/1.0\r\n + Disallow:/\r\n + User-agent: EMail Siphon\r\n + Disallow:/\r\n + User-agent: EMail Wolf\r\n + Disallow:/\r\n User-agent: EasyDL\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Ebingbong\r\n - Disallow: /\r\n - User-agent: eCatch\r\n - Disallow: /\r\n - User-agent: ECCP/1.0\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Ecxi\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: EirGrabber\r\n - Disallow: /\r\n - User-agent: EMail Siphon\r\n - Disallow: /\r\n - User-agent: EMail Wolf\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: EroCrawler\r\n - Disallow: /\r\n - User-agent: evc-batch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Evil\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Exabot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Express WebPictures\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: ExtLinksBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Extractor\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: ExtractorPro\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Extreme Picture Finder\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: EyeNetIE\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Ezooms\r\n - Disallow: /\r\n - User-agent: facebookscraper\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: FDM\r\n - Disallow: /\r\n - User-agent: FemtosearchBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: FHscan\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: FacebookBot\r\n + Disallow:/\r\n + User-agent: FemtosearchBot\r\n + Disallow:/\r\n User-agent: Fimap\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Firefox/7.0\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: FlashGet\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Flunky\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Foobot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Freeuploader\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: FrontPage\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Fuzz\r\n + Disallow:/\r\n User-agent: FyberSpider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Fyrebot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: G-i-g-a-b-o-t\r\n + Disallow:/\r\n + User-agent: GPTBot\r\n + Disallow:/\r\n + User-agent: GT::WWW\r\n + Disallow:/\r\n User-agent: GalaxyBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Genieo\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: GermCrawler\r\n - Disallow: /\r\n - User-agent: Getintent\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: GetRight\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: GetWeb\r\n - Disallow: /\r\n - User-agent: Gigablast\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Getintent\r\n + Disallow:/\r\n User-agent: Gigabot\r\n - Disallow: /\r\n - User-agent: G-i-g-a-b-o-t\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Go!Zilla\r\n + Disallow:/\r\n User-agent: Go-Ahead-Got-It\r\n - Disallow: /\r\n - User-agent: Gotit\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: GoZilla\r\n - Disallow: /\r\n - User-agent: Go!Zilla\r\n - Disallow: /\r\n - User-agent: Grabber\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Google-Extended\r\n + Disallow:/\r\n + User-agent: Gotit\r\n + Disallow:/\r\n User-agent: GrabNet\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Grabber\r\n + Disallow:/\r\n User-agent: Grafula\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: GrapeFX\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: GrapeshotCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: GridBot\r\n - Disallow: /\r\n - User-agent: GT::WWW\r\n - Disallow: /\r\n - User-agent: Haansoft\r\n - Disallow: /\r\n - User-agent: HaosouSpider\r\n - Disallow: /\r\n - User-agent: Harvest\r\n - Disallow: /\r\n - User-agent: Havij\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: HEADMasterSEO\r\n - Disallow: /\r\n - User-agent: heritrix\r\n - Disallow: /\r\n - User-agent: Heritrix\r\n - Disallow: /\r\n - User-agent: Hloader\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: HMView\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: HTMLparser\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: HTTP::Lite\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: HTTrack\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Haansoft\r\n + Disallow:/\r\n + User-agent: HaosouSpider\r\n + Disallow:/\r\n + User-agent: Harvest\r\n + Disallow:/\r\n + User-agent: Havij\r\n + Disallow:/\r\n + User-agent: Heritrix\r\n + Disallow:/\r\n + User-agent: Hloader\r\n + Disallow:/\r\n + User-agent: HonoluluBot\r\n + Disallow:/\r\n User-agent: Humanlinks\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: HybridBot\r\n - Disallow: /\r\n - User-agent: Iblog\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: IDBTE4M\r\n + Disallow:/\r\n User-agent: IDBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: IRLbot\r\n + Disallow:/\r\n + User-agent: Iblog\r\n + Disallow:/\r\n User-agent: Id-search\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: IlseBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Image Fetch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Image Sucker\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: ImagesiftBot\r\n + Disallow:/\r\n User-agent: IndeedBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Indy Library\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: InfoNaviRobot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: InfoTekies\r\n - Disallow: /\r\n - User-agent: instabid\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Information Security Team InfraSec Scanner\r\n + Disallow:/\r\n + User-agent: InfraSec Scanner\r\n + Disallow:/\r\n User-agent: Intelliseek\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: InterGET\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Internet Ninja\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: InternetMeasurement\r\n + Disallow:/\r\n User-agent: InternetSeer\r\n - Disallow: /\r\n - User-agent: internetVista monitor\r\n - Disallow: /\r\n - User-agent: ips-agent\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Iria\r\n - Disallow: /\r\n - User-agent: IRLbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Iskanie\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: IstellaBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: JOC Web Spider\r\n + Disallow:/\r\n User-agent: JamesBOT\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Jbrofuzz\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: JennyBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: JetCar\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Jetty\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: JikeSpider\r\n - Disallow: /\r\n - User-agent: JOC Web Spider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Joomla\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Jorgee\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: JustView\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Jyxobot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Kenjin Spider\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Keybot Translation-Search-Machine\r\n + Disallow:/\r\n User-agent: Keyword Density\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Kinza\r\n + Disallow:/\r\n User-agent: Kozmosbot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: LNSpiderguy\r\n + Disallow:/\r\n + User-agent: LWP::Simple\r\n + Disallow:/\r\n User-agent: Lanshanbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Larbin\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Leap\r\n + Disallow:/\r\n User-agent: LeechFTP\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LeechGet\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LexiBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Lftp\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LibWeb\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Libwhisker\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: LieBaoFast\r\n + Disallow:/\r\n User-agent: Lightspeedsystems\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Likse\r\n - Disallow: /\r\n - User-agent: Linkdexbot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: LinkScan\r\n + Disallow:/\r\n + User-agent: LinkWalker\r\n + Disallow:/\r\n + User-agent: Linkbot\r\n + Disallow:/\r\n User-agent: LinkextractorPro\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LinkpadBot\r\n - Disallow: /\r\n - User-agent: LinkScan\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LinksManager\r\n - Disallow: /\r\n - User-agent: LinkWalker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LinqiaMetadataDownloaderBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LinqiaRSSBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: LinqiaScrapeBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Lipperhey\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Lipperhey Spider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Litemage_walker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Lmspider\r\n - Disallow: /\r\n - User-agent: LNSpiderguy\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Ltx71\r\n - Disallow: /\r\n - User-agent: lwp-request\r\n - Disallow: /\r\n - User-agent: LWP::Simple\r\n - Disallow: /\r\n - User-agent: lwp-trivial\r\n - Disallow: /\r\n - User-agent: Magnet\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: MFC_Tear_Sample\r\n + Disallow:/\r\n + User-agent: MIDown tool\r\n + Disallow:/\r\n + User-agent: MIIxpc\r\n + Disallow:/\r\n + User-agent: MJ12bot\r\n + Disallow:/\r\n + User-agent: MQQBrowser\r\n + Disallow:/\r\n + User-agent: MSFrontPage\r\n + Disallow:/\r\n + User-agent: MSIECrawler\r\n + Disallow:/\r\n + User-agent: MTRobot\r\n + Disallow:/\r\n User-agent: Mag-Net\r\n - Disallow: /\r\n - User-agent: magpie-crawler\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Magnet\r\n + Disallow:/\r\n User-agent: Mail.RU_Bot\r\n - Disallow: /\r\n - User-agent: Majestic12\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Majestic SEO\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Majestic-SEO\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Majestic12\r\n + Disallow:/\r\n User-agent: MarkMonitor\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: MarkWatch\r\n - Disallow: /\r\n - User-agent: Masscan\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Mass Downloader\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Masscan\r\n + Disallow:/\r\n User-agent: Mata Hari\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: MauiBot\r\n - Disallow: /\r\n - User-agent: meanpathbot\r\n - Disallow: /\r\n - User-agent: Meanpathbot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Mb2345Browser\r\n + Disallow:/\r\n User-agent: MeanPath Bot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Meanpathbot\r\n + Disallow:/\r\n User-agent: Mediatoolkitbot\r\n - Disallow: /\r\n - User-agent: mediawords\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: MegaIndex.ru\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Metauri\r\n - Disallow: /\r\n - User-agent: MFC_Tear_Sample\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: MicroMessenger\r\n + Disallow:/\r\n User-agent: Microsoft Data Access\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Microsoft URL Control\r\n - Disallow: /\r\n - User-agent: MIDown tool\r\n - Disallow: /\r\n - User-agent: MIIxpc\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Minefield\r\n + Disallow:/\r\n User-agent: Mister PiX\r\n - Disallow: /\r\n - User-agent: MJ12bot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Moblie Safari\r\n + Disallow:/\r\n User-agent: Mojeek\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Mojolicious\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: MolokaiBot\r\n + Disallow:/\r\n User-agent: Morfeus Fucking Scanner\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Mozlila\r\n + Disallow:/\r\n User-agent: Mr.4x3\r\n - Disallow: /\r\n - User-agent: MSFrontPage\r\n - Disallow: /\r\n - User-agent: MSIECrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Msrabot\r\n - Disallow: /\r\n - User-agent: muhstik-scan\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Musobot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: NICErsPRO\r\n + Disallow:/\r\n + User-agent: NPbot\r\n + Disallow:/\r\n User-agent: Name Intelligence\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Nameprotect\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Navroad\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: NearSite\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Needle\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Nessus\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Net Vampire\r\n + Disallow:/\r\n User-agent: NetAnts\r\n - Disallow: /\r\n - User-agent: Netcraft\r\n - Disallow: /\r\n - User-agent: netEstate NE Crawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: NetLyzer\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: NetMechanic\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: NetSpider\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: NetZIP\r\n + Disallow:/\r\n + User-agent: Netcraft\r\n + Disallow:/\r\n User-agent: Nettrack\r\n - Disallow: /\r\n - User-agent: Net Vampire\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Netvibes\r\n - Disallow: /\r\n - User-agent: NetZIP\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: NextGenSearchBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Nibbler\r\n - Disallow: /\r\n - User-agent: NICErsPRO\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Niki-bot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Nikto\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: NimbleCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Nimbostratus\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Ninja\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Nmap\r\n - Disallow: /\r\n - User-agent: NPbot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Nuclei\r\n + Disallow:/\r\n User-agent: Nutch\r\n - Disallow: /\r\n - User-agent: oBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Octopus\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Offline Explorer\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Offline Navigator\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: OnCrawl\r\n - Disallow: /\r\n - User-agent: Openfind\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: OpenLinkProfiler\r\n - Disallow: /\r\n - User-agent: Openvas\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: OpenVAS\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Openfind\r\n + Disallow:/\r\n + User-agent: Openvas\r\n + Disallow:/\r\n User-agent: OrangeBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: OrangeSpider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: OutclicksBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: OutfoxBot\r\n - Disallow: /\r\n - User-agent: PageAnalyzer\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: PECL::HTTP\r\n + Disallow:/\r\n + User-agent: PHPCrawl\r\n + Disallow:/\r\n + User-agent: POE-Component-Client-HTTP\r\n + Disallow:/\r\n User-agent: Page Analyzer\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: PageAnalyzer\r\n + Disallow:/\r\n User-agent: PageGrabber\r\n - Disallow: /\r\n - User-agent: page scorer\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: PageScorer\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: PageThing.com\r\n + Disallow:/\r\n User-agent: Pandalytics\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Panscient\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Papa Foto\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Pavuk\r\n - Disallow: /\r\n - User-agent: pcBrowser\r\n - Disallow: /\r\n - User-agent: PECL::HTTP\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: PeoplePal\r\n - Disallow: /\r\n - User-agent: PHPCrawl\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Petalbot\r\n + Disallow:/\r\n + User-agent: Pi-Monster\r\n + Disallow:/\r\n User-agent: Picscout\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Picsearch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: PictureFinder\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Piepmatz\r\n + Disallow:/\r\n User-agent: Pimonster\r\n - Disallow: /\r\n - User-agent: Pi-Monster\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Pixray\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: PleaseCrawl\r\n - Disallow: /\r\n - User-agent: plumanalytics\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Pockey\r\n - Disallow: /\r\n - User-agent: POE-Component-Client-HTTP\r\n - Disallow: /\r\n - User-agent: Probethenet\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: ProPowerBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: ProWebWalker\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Probethenet\r\n + Disallow:/\r\n + User-agent: Proximic\r\n + Disallow:/\r\n User-agent: Psbot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Pu_iN\r\n + Disallow:/\r\n User-agent: Pump\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: PxBroker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: PyCurl\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: QueryN Metasearch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Quick-Crawler\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: RSSingBot\r\n + Disallow:/\r\n + User-agent: Rainbot\r\n + Disallow:/\r\n User-agent: RankActive\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RankActiveLinkBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RankFlex\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RankingBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RankingBot2\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Rankivabot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RankurBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Re-re\r\n + Disallow:/\r\n + User-agent: ReGet\r\n + Disallow:/\r\n User-agent: RealDownload\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Reaper\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RebelMouse\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Recorder\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RedesScrapy\r\n - Disallow: /\r\n - User-agent: ReGet\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RepoMonkey\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Ripper\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: RocketCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Rogerbot\r\n - Disallow: /\r\n - User-agent: RSSingBot\r\n - Disallow: /\r\n - User-agent: s1z.ru\r\n - Disallow: /\r\n - User-agent: SalesIntelligent\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SBIder\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: SEOkicks\r\n + Disallow:/\r\n + User-agent: SEOkicks-Robot\r\n + Disallow:/\r\n + User-agent: SEOlyticsCrawler\r\n + Disallow:/\r\n + User-agent: SEOprofiler\r\n + Disallow:/\r\n + User-agent: SEOstats\r\n + Disallow:/\r\n + User-agent: SISTRIX\r\n + Disallow:/\r\n + User-agent: SMTBot\r\n + Disallow:/\r\n + User-agent: SalesIntelligent\r\n + Disallow:/\r\n User-agent: ScanAlert\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Scanbot\r\n - Disallow: /\r\n - User-agent: scan.lol\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: ScoutJet\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Scrapy\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Screaming\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: ScreenerBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: ScrepyBot\r\n + Disallow:/\r\n User-agent: Searchestate\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SearchmetricsBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Seekport\r\n + Disallow:/\r\n + User-agent: SeekportBot\r\n + Disallow:/\r\n + User-agent: SemanticJuice\r\n + Disallow:/\r\n User-agent: Semrush\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SemrushBot\r\n - Disallow: /\r\n - User-agent: SEOkicks\r\n - Disallow: /\r\n - User-agent: SEOkicks-Robot\r\n - Disallow: /\r\n - User-agent: SEOlyticsCrawler\r\n - Disallow: /\r\n - User-agent: Seomoz\r\n - Disallow: /\r\n - User-agent: SEOprofiler\r\n - Disallow: /\r\n - User-agent: seoscanners\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: SentiBot\r\n + Disallow:/\r\n + User-agent: SenutoBot\r\n + Disallow:/\r\n User-agent: SeoSiteCheckup\r\n - Disallow: /\r\n - User-agent: SEOstats\r\n - Disallow: /\r\n - User-agent: serpstatbot\r\n - Disallow: /\r\n - User-agent: sexsearcher\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: SeobilityBot\r\n + Disallow:/\r\n + User-agent: Seomoz\r\n + Disallow:/\r\n User-agent: Shodan\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Siphon\r\n - Disallow: /\r\n - User-agent: SISTRIX\r\n - Disallow: /\r\n - User-agent: Sitebeam\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Site Sucker\r\n + Disallow:/\r\n + User-agent: SiteCheckerBotCrawler\r\n + Disallow:/\r\n User-agent: SiteExplorer\r\n - Disallow: /\r\n - User-agent: Siteimprove\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SiteLockSpider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SiteSnagger\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SiteSucker\r\n - Disallow: /\r\n - User-agent: Site Sucker\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Sitebeam\r\n + Disallow:/\r\n + User-agent: Siteimprove\r\n + Disallow:/\r\n User-agent: Sitevigil\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SlySearch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SmartDownload\r\n - Disallow: /\r\n - User-agent: SMTBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Snake\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Snapbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Snoopy\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SocialRankIOBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sociscraper\r\n - Disallow: /\r\n - User-agent: sogouspider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sogou web spider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sosospider\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sottopop\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SpaceBison\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Spammen\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SpankBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Spanner\r\n - Disallow: /\r\n - User-agent: sp_auditbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Spbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Spinn3r\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SputnikBot\r\n - Disallow: /\r\n - User-agent: spyfu\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sqlmap\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sqlworm\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sqworm\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Steeler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Stripper\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sucker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Sucuri\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SuperBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SuperHTTP\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Surfbot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: SurveyBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Suzuran\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Swiftbot\r\n - Disallow: /\r\n - User-agent: sysscan\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Szukacz\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: T0PHackTeam\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: T8Abot\r\n - Disallow: /\r\n - User-agent: tAkeOut\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Teleport\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: TeleportPro\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Telesoft\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Telesphoreo\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Telesphorep\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: The Intraformant\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: TheNomad\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Thumbor\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: TightTwatBot\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: TinyTestBot\r\n + Disallow:/\r\n User-agent: Titan\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Toata\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Toweyabot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Tracemyfile\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Trendiction\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Trendictionbot\r\n - Disallow: /\r\n - User-agent: trendiction.com\r\n - Disallow: /\r\n - User-agent: trendiction.de\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: True_Robot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Turingos\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Turnitin\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: TurnitinBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: TwengaBot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Twice\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Typhoeus\r\n - Disallow: /\r\n - User-agent: UnisterBot\r\n - Disallow: /\r\n - User-agent: Upflow\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: URLy Warning\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: URLy.Warning\r\n - Disallow: /\r\n - User-agent: Vacuum\r\n - Disallow: /\r\n - User-agent: Vagabondo\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: UnisterBot\r\n + Disallow:/\r\n + User-agent: Upflow\r\n + Disallow:/\r\n + User-agent: V-BOT\r\n + Disallow:/\r\n User-agent: VB Project\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: VCI\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Vacuum\r\n + Disallow:/\r\n + User-agent: Vagabondo\r\n + Disallow:/\r\n + User-agent: VelenPublicWebCrawler\r\n + Disallow:/\r\n User-agent: VeriCiteCrawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: VidibleScraper\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Virusdie\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: VoidEYE\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Voil\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Voltron\r\n - Disallow: /\r\n - User-agent: Wallpapers/3.0\r\n - Disallow: /\r\n - User-agent: WallpapersHD\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WASALive-Bot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WBSearchBot\r\n - Disallow: /\r\n - User-agent: Webalta\r\n - Disallow: /\r\n - User-agent: WebAuto\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: WEBDAV\r\n + Disallow:/\r\n + User-agent: WISENutbot\r\n + Disallow:/\r\n + User-agent: WPScan\r\n + Disallow:/\r\n + User-agent: WWW-Collector-E\r\n + Disallow:/\r\n + User-agent: WWW-Mechanize\r\n + Disallow:/\r\n + User-agent: WWW::Mechanize\r\n + Disallow:/\r\n + User-agent: WWWOFFLE\r\n + Disallow:/\r\n + User-agent: Wallpapers\r\n + Disallow:/\r\n + User-agent: Wallpapers/3.0\r\n + Disallow:/\r\n + User-agent: WallpapersHD\r\n + Disallow:/\r\n + User-agent: WeSEE\r\n + Disallow:/\r\n User-agent: Web Auto\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Web Collage\r\n + Disallow:/\r\n + User-agent: Web Enhancer\r\n + Disallow:/\r\n + User-agent: Web Fetch\r\n + Disallow:/\r\n + User-agent: Web Fuck\r\n + Disallow:/\r\n + User-agent: Web Pix\r\n + Disallow:/\r\n + User-agent: Web Sauger\r\n + Disallow:/\r\n + User-agent: Web Sucker\r\n + Disallow:/\r\n + User-agent: WebAuto\r\n + Disallow:/\r\n User-agent: WebBandit\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebCollage\r\n - Disallow: /\r\n - User-agent: Web Collage\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebCopier\r\n - Disallow: /\r\n - User-agent: WEBDAV\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebEnhancer\r\n - Disallow: /\r\n - User-agent: Web Enhancer\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebFetch\r\n - Disallow: /\r\n - User-agent: Web Fetch\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebFuck\r\n - Disallow: /\r\n - User-agent: Web Fuck\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebGo IS\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebImageCollector\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebLeacher\r\n - Disallow: /\r\n - User-agent: WebmasterWorldForumBot\r\n - Disallow: /\r\n - User-agent: webmeup-crawler\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebPix\r\n - Disallow: /\r\n - User-agent: Web Pix\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebReaper\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebSauger\r\n - Disallow: /\r\n - User-agent: Web Sauger\r\n - Disallow: /\r\n - User-agent: Webshag\r\n - Disallow: /\r\n - User-agent: WebsiteExtractor\r\n - Disallow: /\r\n - User-agent: WebsiteQuester\r\n - Disallow: /\r\n - User-agent: Website Quester\r\n - Disallow: /\r\n - User-agent: Webster\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebStripper\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebSucker\r\n - Disallow: /\r\n - User-agent: Web Sucker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebWhacker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WebZIP\r\n - Disallow: /\r\n - User-agent: WeSEE\r\n - Disallow: /\r\n + Disallow:/\r\n + User-agent: Webalta\r\n + Disallow:/\r\n + User-agent: WebmasterWorldForumBot\r\n + Disallow:/\r\n + User-agent: Webshag\r\n + Disallow:/\r\n + User-agent: Website Quester\r\n + Disallow:/\r\n + User-agent: WebsiteExtractor\r\n + Disallow:/\r\n + User-agent: WebsiteQuester\r\n + Disallow:/\r\n + User-agent: Webster\r\n + Disallow:/\r\n User-agent: Whack\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Whacker\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Whatweb\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Who.is Bot\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: Widow\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WinHTTrack\r\n - Disallow: /\r\n + Disallow:/\r\n User-agent: WiseGuys Robot\r\n + Disallow:/\r\n + User-agent: Wonderbot\r\n + Disallow:/\r\n + User-agent: Woobot\r\n + Disallow:/\r\n + User-agent: Wotbox\r\n + Disallow:/\r\n + User-agent: Wprecon\r\n + Disallow:/\r\n + User-agent: Xaldon WebSpider\r\n + Disallow:/\r\n + User-agent: Xaldon_WebSpider\r\n + Disallow:/\r\n + User-agent: Xenu\r\n + Disallow:/\r\n + User-agent: YaK\r\n + Disallow:/\r\n + User-agent: YoudaoBot\r\n + Disallow:/\r\n + User-agent: Zade\r\n + Disallow:/\r\n + User-agent: Zauba\r\n + Disallow:/\r\n + User-agent: Zermelo\r\n + Disallow:/\r\n + User-agent: Zeus\r\n + Disallow:/\r\n + User-agent: Zitebot\r\n + Disallow:/\r\n + User-agent: ZmEu\r\n + Disallow:/\r\n + User-agent: ZoomBot\r\n + Disallow:/\r\n + User-agent: ZoominfoBot\r\n + Disallow:/\r\n + User-agent: ZumBot\r\n + Disallow:/\r\n + User-agent: ZyBorg\r\n + Disallow:/\r\n + User-agent: adscanner\r\n + Disallow:/\r\n + User-agent: anthropic-ai\r\n + Disallow:/\r\n + User-agent: archive.org_bot\r\n + Disallow:/\r\n + User-agent: arquivo-web-crawler\r\n + Disallow:/\r\n + User-agent: arquivo.pt\r\n + Disallow:/\r\n + User-agent: autoemailspider\r\n + Disallow:/\r\n + User-agent: awario.com\r\n + Disallow:/\r\n + User-agent: backlink-check\r\n + Disallow:/\r\n + User-agent: cah.io.community\r\n + Disallow:/\r\n + User-agent: check1.exe\r\n + Disallow:/\r\n + User-agent: clark-crawler\r\n + Disallow:/\r\n + User-agent: coccocbot\r\n + Disallow:/\r\n + User-agent: cognitiveseo\r\n + Disallow:/\r\n + User-agent: cohere-ai\r\n + Disallow:/\r\n + User-agent: com.plumanalytics\r\n + Disallow:/\r\n + User-agent: crawl.sogou.com\r\n + Disallow:/\r\n + User-agent: crawler.feedback\r\n + Disallow:/\r\n + User-agent: crawler4j\r\n + Disallow:/\r\n + User-agent: dataforseo.com\r\n + Disallow:/\r\n + User-agent: dataforseobot\r\n + Disallow:/\r\n + User-agent: demandbase-bot\r\n + Disallow:/\r\n + User-agent: domainsproject.org\r\n + Disallow:/\r\n + User-agent: eCatch\r\n + Disallow:/\r\n + User-agent: evc-batch\r\n + Disallow:/\r\n + User-agent: facebookscraper\r\n + Disallow:/\r\n + User-agent: gopher\r\n + Disallow:/\r\n + User-agent: heritrix\r\n + Disallow:/\r\n + User-agent: imagesift.com\r\n + Disallow:/\r\n + User-agent: instabid\r\n + Disallow:/\r\n + User-agent: internetVista monitor\r\n + Disallow:/\r\n + User-agent: ips-agent\r\n + Disallow:/\r\n + User-agent: isitwp.com\r\n + Disallow:/\r\n + User-agent: iubenda-radar\r\n + Disallow:/\r\n + User-agent: linkdexbot\r\n + Disallow:/\r\n + User-agent: linkfluence\r\n + Disallow:/\r\n + User-agent: lwp-request\r\n + Disallow:/\r\n + User-agent: lwp-trivial\r\n + Disallow:/\r\n + User-agent: magpie-crawler\r\n + Disallow:/\r\n + User-agent: meanpathbot\r\n + Disallow:/\r\n + User-agent: mediawords\r\n + Disallow:/\r\n + User-agent: muhstik-scan\r\n + Disallow:/\r\n + User-agent: netEstate NE Crawler\r\n + Disallow:/\r\n + User-agent: oBot\r\n + Disallow:/\r\n + User-agent: omgili\r\n + Disallow:/\r\n + User-agent: openai\r\n + Disallow:/\r\n + User-agent: openai.com\r\n + Disallow:/\r\n + User-agent: page scorer\r\n + Disallow:/\r\n + User-agent: pcBrowser\r\n + Disallow:/\r\n + User-agent: plumanalytics\r\n + Disallow:/\r\n + User-agent: polaris version\r\n + Disallow:/\r\n + User-agent: probe-image-size\r\n + Disallow:/\r\n + User-agent: ripz\r\n + Disallow:/\r\n + User-agent: s1z.ru\r\n + Disallow:/\r\n + User-agent: satoristudio.net\r\n + Disallow:/\r\n + User-agent: scalaj-http\r\n + Disallow:/\r\n + User-agent: scan.lol\r\n + Disallow:/\r\n + User-agent: seobility\r\n + Disallow:/\r\n + User-agent: seocompany.store\r\n + Disallow:/\r\n + User-agent: seoscanners\r\n + Disallow:/\r\n + User-agent: seostar\r\n + Disallow:/\r\n + User-agent: serpstatbot\r\n + Disallow:/\r\n + User-agent: sexsearcher\r\n + Disallow:/\r\n + User-agent: sitechecker.pro\r\n + Disallow:/\r\n + User-agent: siteripz\r\n + Disallow:/\r\n + User-agent: sogouspider\r\n + Disallow:/\r\n + User-agent: sp_auditbot\r\n + Disallow:/\r\n + User-agent: spyfu\r\n + Disallow:/\r\n + User-agent: sysscan\r\n + Disallow:/\r\n + User-agent: tAkeOut\r\n + Disallow:/\r\n + User-agent: trendiction.com\r\n + Disallow:/\r\n + User-agent: trendiction.de\r\n + Disallow:/\r\n + User-agent: ubermetrics-technologies.com\r\n + Disallow:/\r\n + User-agent: voyagerx.com\r\n + Disallow:/\r\n + User-agent: webgains-bot\r\n + Disallow:/\r\n + User-agent: webmeup-crawler\r\n + Disallow:/\r\n + User-agent: webpros.com\r\n + Disallow:/\r\n + User-agent: webprosbot\r\n + Disallow:/\r\n + User-agent: x09Mozilla\r\n + Disallow:/\r\n + User-agent: x22Mozilla\r\n + Disallow:/\r\n + User-agent: xpymep1.exe\r\n + Disallow:/\r\n + User-agent: zauba.io\r\n + Disallow:/\r\n + User-agent: zgrab\r\n + Disallow:/\r\n + \r\n + # Dark Visitors Robots.txt\r\n + User-agent: GPTBot\r\n Disallow: /\r\n - User-agent: WISENutbot\r\n + User-agent: ChatGPT-User\r\n Disallow: /\r\n - User-agent: Wonderbot\r\n + User-agent: Applebot-Extended\r\n Disallow: /\r\n - User-agent: Woobot\r\n + User-agent: Bytespider\r\n Disallow: /\r\n - User-agent: Wotbox\r\n + User-agent: CCBot\r\n Disallow: /\r\n - User-agent: Wprecon\r\n + User-agent: ClaudeBot\r\n Disallow: /\r\n - User-agent: WPScan\r\n + User-agent: Diffbot\r\n Disallow: /\r\n - User-agent: WWW-Collector-E\r\n + User-agent: FacebookBot\r\n Disallow: /\r\n - User-agent: WWW-Mechanize\r\n + User-agent: Google-Extended\r\n Disallow: /\r\n - User-agent: WWW::Mechanize\r\n + User-agent: Meta-ExternalAgent\r\n Disallow: /\r\n - User-agent: WWWOFFLE\r\n + User-agent: omgili\r\n Disallow: /\r\n - User-agent: x09Mozilla\r\n + User-agent: omgilibot\r\n Disallow: /\r\n - User-agent: x22Mozilla\r\n + User-agent: Timpibot\r\n Disallow: /\r\n - User-agent: Xaldon WebSpider\r\n + User-agent: anthropic-ai\r\n Disallow: /\r\n - User-agent: Xaldon_WebSpider\r\n + User-agent: Claude-Web\r\n Disallow: /\r\n - User-agent: Xenu\r\n + User-agent: cohere-ai\r\n Disallow: /\r\n - User-agent: xpymep1.exe\r\n + User-agent: DataForSeoBot\r\n Disallow: /\r\n - User-agent: YoudaoBot\r\n + User-agent: FriendlyCrawler\r\n Disallow: /\r\n - User-agent: Zade\r\n + User-agent: img2dataset\r\n Disallow: /\r\n - User-agent: Zauba\r\n + User-agent: ImagesiftBot\r\n Disallow: /\r\n - User-agent: zauba.io\r\n + User-agent: Meltwater\r\n Disallow: /\r\n - User-agent: Zermelo\r\n + User-agent: peer39_crawler\r\n Disallow: /\r\n - User-agent: Zeus\r\n + User-agent: peer39_crawler/1.0\r\n Disallow: /\r\n - User-agent: zgrab\r\n + User-agent: PerplexityBot\r\n Disallow: /\r\n - User-agent: Zitebot\r\n + User-agent: PiplBot\r\n Disallow: /\r\n - User-agent: ZmEu\r\n + User-agent: scoop.it\r\n Disallow: /\r\n - User-agent: ZumBot\r\n + User-agent: Seekr\r\n Disallow: /\r\n - User-agent: ZyBorg\r\n + User-agent: YouBot\r\n Disallow: /\r\n \r\n ### CUSTOM:\r\n @@ -1164,5 +1420,7 @@ routes: Disallow: /print/\r\n Disallow: /login/\r\n Disallow: /logout/\r\n + Disallow: /private/\r\n + Disallow: /config/\r\n \r\n Sitemap: https://www.{{EXTENSION_DOMAIN_NAME}}.{{EXTENSION_DOMAIN_TLD}}/sitemap.xml"