added noindex headers

challenger/src/main/java/uk/co/compendiumdev/challenge/apimodel/ChallengeThingifier.java

@@ -27,6 +27,7 @@ private void createThingifier(){
         this.challengeThingifier.apiConfig().setApiToShowPrimaryKeyHeaderInResponse(true);
         this.challengeThingifier.apiConfig().forParams().willEnforceFilteringThroughUrlParams();
         this.challengeThingifier.apiConfig().forParams().willAllowFilteringThroughUrlParams();
+        this.challengeThingifier.apiConfig().setApiToAllowRobotsIndexingResponses(false);
 
         this.challengeDefn = this.challengeThingifier.defineThing(
                                     "challenge", "challenges");

challenger/src/main/java/uk/co/compendiumdev/challenge/challengesrouting/SimulationRoutes.java

@@ -90,7 +90,7 @@ public void configure(final ThingifierApiDefn apiDefn) {
 
         options(endpoint + "/*", (request, result) -> {
             result.status(204);
-            result.header("X-Robots-Tag", "noindex");
+            result.header("x-robots-tag", "noindex");
             result.header("Allow", "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS, TRACE");
             return "";
         });

challenger/src/main/java/uk/co/compendiumdev/challenge/gui/ChallengerWebGUI.java

@@ -444,6 +444,10 @@ public void setup(final Challengers challengers,
 
                     response.body(html.toString());
                     response.type("text/html");
+                    if(response.raw().containsHeader("x-robots-tag")){
+                        // we want it indexed because it is content
+                        response.raw().setHeader("x-robots-tag", "all");
+                    }
                     response.status(200);
                 }
 

challenger/src/main/resources/content/practice-sites.md

@@ -43,7 +43,7 @@ There are many API simulators so we have only listed a few of the best here.
 
 #### JSON Placeholder
 
-[{JSON} Placeholder]https://jsonplaceholder.typicode.com/
+[{JSON} Placeholder](https://jsonplaceholder.typicode.com/)
 
 - An API For `GET` requests.
 - Acts as a simulator for Update and Delete requests
@@ -116,7 +116,7 @@ These are full APIs online, which means you don't have to install anything and c
 
 ---
 
-## Toolshop API
+#### Toolshop API
 
 [Toolshop API](https://api.practicesoftwaretesting.com/api/documentation)
 
@@ -218,7 +218,7 @@ I found Reqres a little hard to understand from the documentation. It seems as t
 - [reqres](https://reqres.in)
     - Supports `JSON` only
   - Has hard coded data.
-  - Provides a Swagger [API interface and docmentation](https://reqres.in/api-docs/) this can be useful to get started but doesn't cover the generic endpoints so you'll need to use a REST Client for that.
+  - Provides a Swagger [API interface and documentation](https://reqres.in/api-docs/) this can be useful to get started but doesn't cover the generic endpoints so you'll need to use a REST Client for that.
   - There is also a hard coded set of 12 'things' which will respond to any endpoint e.g. `GET https://reqres/in/api/books/1` will return the hardcoded 'thing' with ID 1 (it won't be a 'book' but the API responds as though the `/api/books/` end point exists)
   - Simulates `DELETE, PUT, POST, PATCH` operations.
 

challenger/src/main/resources/public/robots.txt

@@ -11,10 +11,11 @@ Disallow: /mirror/request
 Disallow: /mirror/raw/*
 Disallow: /mirror/request/*
 Disallow: /sim/*
-Disallow: /todos/*
-Disallow: /todos
-Disallow: /challenges
 Disallow: /challenger
 Disallow: /challenger/*
 Disallow: /secret/*
-Disallow: /heartbeat
+# use noindex http header instead
+# Disallow: /todos/*
+# Disallow: /todos
+# Disallow: /challenges
+# Disallow: /heartbeat

thingifier/src/main/java/uk/co/compendiumdev/thingifier/api/http/HttpApiResponse.java

@@ -65,9 +65,9 @@ private void configureFrom(final HttpHeadersBlock requestHeaders, final HttpHead
         apiResponseHeaders.putAll(originalApiResponseHeaders);
         apiResponseHeaders.put("Content-Type", type);
 
-        // todo: put this behind a config
+
         if(apiConfig.willPreventRobotsFromIndexingResponse()) {
-            apiResponseHeaders.put("X-Robots-Tag", "noindex");
+            apiResponseHeaders.put("x-robots-tag", "noindex");
         }
 
     }

thingifier/src/main/java/uk/co/compendiumdev/thingifier/apiconfig/ThingifierApiConfig.java

@@ -76,7 +76,7 @@ public ThingifierApiConfig(){
         willApiEnforceContentTypeHeaderForRequests = true;
         acceptXmlContent = true;
         acceptJsonContent = true;
-        robotsCanIndexApiResponses=false;
+        robotsCanIndexApiResponses=true;
 
         paramsConfig = new ParamConfig();
 
@@ -100,7 +100,7 @@ public void setFrom(final ThingifierApiConfig apiConfig) {
         robotsCanIndexApiResponses = apiConfig.willAllowRobotsToIndexResponses();
         willApiAllowXmlResponses = apiConfig.willApiAllowXmlForResponses();
         willApiAllowJsonResponses = apiConfig.willApiAllowJsonForResponses();
-        willApiEnforceAcceptHeaderForResponses = willApiEnforceAcceptHeaderForResponses();
+        willApiEnforceAcceptHeaderForResponses = apiConfig.willApiEnforceAcceptHeaderForResponses();
 
         paramsConfig.setFrom(apiConfig.forParams());
         statusCodeConfig.setFrom(apiConfig.statusCodes());

thingifier/src/main/java/uk/co/compendiumdev/thingifier/spark/SimpleRouteConfig.java

@@ -61,7 +61,7 @@ public static void routeStatus(final int statuscode, final String endpoint, fina
                     preferred="application/json"; // hard coded default
                 }
                 result.header("Content-Type", preferred);
-                result.header("X-Robots-Tag", "noindex");
+                result.header("x-robots-tag", "noindex");
                 result.status(statuscode);
                 return "";
             };