Application health status check

ci/validate-container

@@ -23,7 +23,7 @@ fi
 docker run --detach --rm --name $container_name camptocamp/mapfish_print:latest
 sleep 15
 http_code=$(docker exec $container_name curl -sL -w "%{http_code}" -I localhost:8080/metrics/healthcheck -o /dev/null)
-if [[ $http_code = 501 ]]; then
+if [[ $http_code = 200 ]]; then
     echo "container healthy"
     cleanup
     exit 0

core/src/main/java/org/mapfish/print/metrics/ApplicationStatus.java

@@ -0,0 +1,68 @@
+package org.mapfish.print.metrics;
+
+import com.codahale.metrics.health.HealthCheck;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import org.mapfish.print.servlet.job.JobQueue;
+import org.mapfish.print.servlet.job.impl.ThreadPoolJobManager;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+class ApplicationStatus extends HealthCheck {
+  @Value("${healthStatus.expectedMaxTime.sinceLastPrint.InSeconds}")
+  private int secondsInFloatingWindow;
+
+  @Value("${healthStatus.unhealthyThreshold.maxNbrPrintJobQueued}")
+  private int maxNbrPrintJobQueued;
+
+  @Autowired private JobQueue jobQueue;
+  @Autowired private ThreadPoolJobManager jobManager;
+
+  /**
+   * When a Result is returned it can be healthy or unhealthy. In both cases it is associated to a
+   * Http 200 status. When an exception is thrown it means the server is no longer working at all,
+   * it is associated to a Http 500 status.
+   */
+  @Override
+  protected Result check() throws Exception {
+    long waitingJobsCount = jobQueue.getWaitingJobsCount();
+    if (waitingJobsCount == 0) {
+      return Result.healthy("No print job is waiting in the queue.");
+    }
+
+    String health = ". Number of print jobs waiting is " + waitingJobsCount;
+
+    if (jobManager.getLastExecutedJobTimestamp() == null) {
+      return Result.unhealthy("No print job was ever processed by this server" + health);
+    } else if (hasThisServerPrintedRecently()) {
+      // WIP (See issue https://github.com/mapfish/mapfish-print/issues/3393)
+      if (waitingJobsCount > maxNbrPrintJobQueued) {
+        return Result.unhealthy(
+            "WIP: Number of print jobs queued is above threshold: "
+                + maxNbrPrintJobQueued
+                + health);
+      } else {
+        return Result.healthy("This server instance is printing" + health);
+      }
+    } else {
+      throw notificationForBrokenServer();
+    }
+  }
+
+  private RuntimeException notificationForBrokenServer() {
+    return new RuntimeException(
+        "None of the print job queued was processed by this server, in the last (seconds): "
+            + secondsInFloatingWindow);
+  }
+
+  private boolean hasThisServerPrintedRecently() {
+    final Instant lastExecutedJobTime = jobManager.getLastExecutedJobTimestamp().toInstant();
+    final Instant beginningOfTimeWindow = getBeginningOfTimeWindow();
+    return lastExecutedJobTime.isAfter(beginningOfTimeWindow);
+  }
+
+  private Instant getBeginningOfTimeWindow() {
+    return new Date().toInstant().minus(Duration.ofSeconds(secondsInFloatingWindow));
+  }
+}

core/src/main/java/org/mapfish/print/metrics/HealthCheckingRegistry.java

@@ -0,0 +1,13 @@
+package org.mapfish.print.metrics;
+
+import javax.annotation.PostConstruct;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class HealthCheckingRegistry extends com.codahale.metrics.health.HealthCheckRegistry {
+  @Autowired private ApplicationStatus applicationStatus;
+
+  @PostConstruct
+  public void registerHealthCheck() {
+    register("application", applicationStatus);
+  }
+}

core/src/main/java/org/mapfish/print/servlet/job/JobManager.java

@@ -1,8 +1,9 @@
 package org.mapfish.print.servlet.job;
 
+import java.util.Date;
+
 /** Manages and Executes Print Jobs. */
 public interface JobManager {
-
   /**
    * Submit a new job for execution.
    *
@@ -14,15 +15,22 @@ public interface JobManager {
    * Cancel a job.
    *
    * @param referenceId The referenceId of the job to cancel.
-   * @throws NoSuchReferenceException
+   * @throws NoSuchReferenceException When trying to cancel an unknown referenceId
    */
   void cancel(String referenceId) throws NoSuchReferenceException;
 
   /**
    * Get the status for a job.
    *
    * @param referenceId The referenceId of the job to check.
-   * @throws NoSuchReferenceException
+   * @throws NoSuchReferenceException When requesting status of an unknown referenceId.
    */
   PrintJobStatus getStatus(String referenceId) throws NoSuchReferenceException;
+
+  /**
+   * Instant at which a job was executed by this manager.
+   *
+   * @return the timestamp as a Date.
+   */
+  Date getLastExecutedJobTimestamp();
 }

core/src/main/java/org/mapfish/print/servlet/job/impl/ThreadPoolJobManager.java

@@ -6,6 +6,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -118,6 +119,7 @@ public class ThreadPoolJobManager implements JobManager {
   @Autowired private MetricRegistry metricRegistry;
 
   private boolean requestedToStop = false;
+  private Date lastExecutedJobTimestamp;
 
   public final void setMaxNumberOfRunningPrintJobs(final int maxNumberOfRunningPrintJobs) {
     this.maxNumberOfRunningPrintJobs = maxNumberOfRunningPrintJobs;
@@ -290,7 +292,12 @@ public final void shutdown() {
     }
   }
 
+  public Date getLastExecutedJobTimestamp() {
+    return lastExecutedJobTimestamp;
+  }
+
   private void executeJob(final PrintJob job) {
+    lastExecutedJobTimestamp = new Date();
     LOGGER.debug(
         "executeJob {}, PoolSize {}, CorePoolSize {}, Active {}, Completed {}, Task {}, isShutdown"
             + " {}, isTerminated {}",

core/src/main/resources/mapfish-spring-application-context.xml

@@ -57,7 +57,8 @@
     <bean id="fileReportLoader" class="org.mapfish.print.servlet.job.loader.FileReportLoader"/>
 
     <bean id="metricRegistry" class="com.codahale.metrics.MetricRegistry"/>
-    <bean id="healthCheckRegistry" class="com.codahale.metrics.health.HealthCheckRegistry"/>
+    <bean id="applicationStatus" class="org.mapfish.print.metrics.ApplicationStatus"/>
+    <bean id="healthCheckRegistry" class="org.mapfish.print.metrics.HealthCheckingRegistry"/>
     <bean id="httpClientFactory" class="org.mapfish.print.http.MfClientHttpRequestFactoryImpl">
         <constructor-arg index="0" value="${maxConnectionsTotal}" />
         <constructor-arg index="1" value="${maxConnectionsPerRoute}" />

core/src/main/resources/mapfish-spring.properties

@@ -52,3 +52,9 @@ httpRequest.fetchRetry.maxNumber=3
 
 # Number of milliseconds between 2 executions of the same request
 httpRequest.fetchRetry.intervalMillis=100
+
+# Amount of time in the past where we check if a print job was executed by this server
+healthStatus.expectedMaxTime.sinceLastPrint.InSeconds=300
+
+# Maximum number of Print Jobs queued before raising it i
+healthStatus.unhealthyThreshold.maxNbrPrintJobQueued=4

core/src/test/java/org/mapfish/print/metrics/ApplicationStatusTest.java

@@ -0,0 +1,86 @@
+package org.mapfish.print.metrics;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrowsExactly;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+import com.codahale.metrics.health.HealthCheck;
+import java.util.Date;
+import org.junit.Before;
+import org.junit.Test;
+import org.mapfish.print.AbstractMapfishSpringTest;
+import org.mapfish.print.servlet.job.JobQueue;
+import org.mapfish.print.servlet.job.impl.ThreadPoolJobManager;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class ApplicationStatusTest extends AbstractMapfishSpringTest {
+
+  @Mock private JobQueue jobQueue;
+
+  @Mock private ThreadPoolJobManager jobManager;
+
+  @Autowired @InjectMocks private ApplicationStatus applicationStatus;
+
+  @Before
+  public void setUp() throws Exception {
+    // Initialize mocks created above
+    MockitoAnnotations.openMocks(this);
+  }
+
+  @Test
+  public void testCheck_Success_NoPrintJobs() throws Exception {
+    when(jobQueue.getWaitingJobsCount()).thenReturn(0L);
+
+    HealthCheck.Result result = applicationStatus.check();
+
+    assertTrue(result.isHealthy());
+    assertEquals("No print job is waiting in the queue.", result.getMessage());
+  }
+
+  @Test
+  public void testCheck_Failed_NoPrintJobs() throws Exception {
+    when(jobQueue.getWaitingJobsCount()).thenReturn(1L);
+    when(jobManager.getLastExecutedJobTimestamp()).thenReturn(new Date(0L));
+
+    RuntimeException rte =
+        assertThrowsExactly(
+            RuntimeException.class,
+            () -> {
+              // WHEN
+              applicationStatus.check();
+            });
+
+    assertEquals(
+        "None of the print job queued was processed by this server, in the last (seconds): 300",
+        rte.getMessage());
+  }
+
+  @Test
+  public void testCheck_Success_PrintJobs() throws Exception {
+    when(jobQueue.getWaitingJobsCount()).thenReturn(5L, 4L);
+    when(jobManager.getLastExecutedJobTimestamp()).thenReturn(new Date());
+
+    applicationStatus.check();
+    HealthCheck.Result result = applicationStatus.check();
+
+    assertTrue(result.isHealthy());
+    assertTrue(result.getMessage().contains("This server instance is printing."));
+  }
+
+  @Test
+  public void testCheck_Fail_TooManyJobsAreQueued() throws Exception {
+    when(jobQueue.getWaitingJobsCount()).thenReturn(4L, 5L);
+    when(jobManager.getLastExecutedJobTimestamp()).thenReturn(new Date());
+
+    applicationStatus.check();
+    HealthCheck.Result result = applicationStatus.check();
+
+    assertFalse(result.isHealthy());
+    assertTrue(result.getMessage().contains("Number of print jobs queued is above threshold: "));
+  }
+}

examples/src/test/java/org/mapfish/print/MetricsApiTest.java

@@ -2,6 +2,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -26,7 +27,7 @@ public class MetricsApiTest extends AbstractApiTest {
 
   @Test
   public void testMetrics() throws Exception {
-    ClientHttpRequest request = getMetricsRequest("metrics", HttpMethod.GET);
+    ClientHttpRequest request = getMetricsRequest("metrics");
     try (ClientHttpResponse response = request.execute()) {
       assertEquals(HttpStatus.OK, response.getStatusCode());
       assertEquals(MediaType.APPLICATION_JSON, response.getHeaders().getContentType());
@@ -37,7 +38,7 @@ public void testMetrics() throws Exception {
 
   @Test
   public void testPing() throws Exception {
-    ClientHttpRequest request = getMetricsRequest("ping", HttpMethod.GET);
+    ClientHttpRequest request = getMetricsRequest("ping");
     try (ClientHttpResponse response = request.execute()) {
       assertEquals(HttpStatus.OK, response.getStatusCode());
       assertEquals("pong", getBodyAsText(response).trim());
@@ -46,7 +47,7 @@ public void testPing() throws Exception {
 
   @Test
   public void testThreads() throws Exception {
-    ClientHttpRequest request = getMetricsRequest("threads", HttpMethod.GET);
+    ClientHttpRequest request = getMetricsRequest("threads");
     try (ClientHttpResponse response = request.execute()) {
       assertEquals(HttpStatus.OK, response.getStatusCode());
       assertEquals(MediaType.TEXT_PLAIN, response.getHeaders().getContentType());
@@ -56,18 +57,19 @@ public void testThreads() throws Exception {
 
   @Test
   public void testHealthcheck() throws Exception {
-    ClientHttpRequest request = getMetricsRequest("healthcheck", HttpMethod.GET);
+    ClientHttpRequest request = getMetricsRequest("healthcheck");
     try (ClientHttpResponse response = request.execute()) {
-      // TODO not implemented?
-      assertEquals(HttpStatus.NOT_IMPLEMENTED, response.getStatusCode());
-      //        assertEquals(HttpStatus.OK, response.getStatusCode());
-      //        assertEquals(MediaType.APPLICATION_JSON, response.getHeaders().getContentType());
-      //        assertNotNull(new JSONObject(getBodyAsText(response)));
+      assertEquals(HttpStatus.OK, response.getStatusCode());
+      assertEquals(MediaType.APPLICATION_JSON, response.getHeaders().getContentType());
+      String bodyAsText = getBodyAsText(response);
+      assertNotNull(bodyAsText);
+      JSONObject healthcheck = new JSONObject(bodyAsText);
+      JSONObject application = healthcheck.getJSONObject("application");
+      assertTrue(application.getBoolean("healthy"));
     }
   }
 
-  private ClientHttpRequest getMetricsRequest(String path, HttpMethod method)
-      throws IOException, URISyntaxException {
-    return getRequest("metrics/" + path, method);
+  private ClientHttpRequest getMetricsRequest(String path) throws IOException, URISyntaxException {
+    return getRequest("metrics/" + path, HttpMethod.GET);
   }
 }

examples/src/test/java/org/mapfish/print/PrintApiTest.java

@@ -44,7 +44,7 @@ public void testListApps() throws Exception {
       assertEquals(HttpStatus.OK, response.getStatusCode());
       assertEquals(getJsonMediaType(), response.getHeaders().getContentType());
       final JSONArray appIdsJson = new JSONArray(getBodyAsText(response));
-      assertTrue(appIdsJson.length() > 0);
+      assertFalse(appIdsJson.isEmpty());
     }
   }