Modify robots.txt for the warper relative url root

nypl-spacetime · Jul 6, 2015 · 4bd536d · 4bd536d
1 parent 1ad1198
commit 4bd536d
Showing 1 changed file with 11 additions and 11 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,14 +1,14 @@
 # See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
 #
 # To ban all spiders from the entire site uncomment the next two lines:
- User-Agent: *
- Crawl-delay: 3
- Allow: /$
- Allow: /maps$
- Allow: /maps/$
- Allow: /layers/$
- Allow: /layers$
- Allow: /home
- Disallow: /*
- Disallow: /maps/*
- Disallow: /layers/*
+User-Agent: *
+Crawl-delay: 3
+Allow: /warper/$
+Allow: /warper/maps$
+Allow: /warper/maps/$
+Allow: /warper/layers/$
+Allow: /warper/layers$
+Allow: /warper/home
+Disallow: /warper/*
+Disallow: /warper/maps/*
+Disallow: /warper/layers/*