fix: Fix handling unavailable services (#3879)

Signed-off-by: Pavel Jareš <[email protected]>
Signed-off-by: ac892247 <[email protected]>
Signed-off-by: Pavel Jareš <[email protected]>
Co-authored-by: ac892247 <[email protected]>

.github/workflows/integration-tests.yml

@@ -1420,6 +1420,14 @@ jobs:
             -   name: Run startup check
                 run: >
                     ./gradlew runStartUpCheck --info --scan -Denvironment.config=-ha -Ddiscoverableclient.instances=1
+            -   name: Show status when APIML is not ready yet
+                if: failure()
+                shell: bash
+                run: |
+                    apt update
+                    apt install -y apt-transport-https ca-certificates curl software-properties-common
+                    curl -k -s https://gateway-service:10010/application/health
+                    curl -k -s https://gateway-service-2:10010/application/health
             -   name: Cypress run API Catalog
                 run: |
                     cd api-catalog-ui/frontend

gateway-service/src/main/java/org/zowe/apiml/gateway/config/NettyRoutingFilterApiml.java

@@ -15,12 +15,16 @@
 import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cloud.gateway.config.HttpClientProperties;
+import org.springframework.cloud.gateway.filter.GatewayFilterChain;
 import org.springframework.cloud.gateway.filter.NettyRoutingFilter;
 import org.springframework.cloud.gateway.filter.headers.HttpHeadersFilter;
 import org.springframework.cloud.gateway.route.Route;
 import org.springframework.web.server.ServerWebExchange;
+import org.zowe.apiml.security.common.error.ServiceNotAccessibleException;
+import reactor.core.publisher.Mono;
 import reactor.netty.http.client.HttpClient;
 
+import java.net.ConnectException;
 import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
@@ -81,4 +85,14 @@ protected HttpClient getHttpClient(Route route, ServerWebExchange exchange) {
             .responseTimeout(Duration.ofMillis(requestTimeout));
     }
 
+    @Override
+    public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
+        return super.filter(exchange, chain).onErrorResume(e -> {
+            if (e.getCause() instanceof ConnectException) {
+                var uri = exchange.getRequest().getURI();
+                return Mono.error(new ServiceNotAccessibleException(String.format("Service is not available at %s://%s:%d", uri.getScheme(), uri.getHost(), uri.getPort()), e));
+            }
+            return Mono.error(e);
+        });
+    }
 }

gateway-service/src/main/java/org/zowe/apiml/gateway/controllers/GatewayExceptionHandler.java

@@ -33,6 +33,7 @@
 import org.springframework.web.server.session.DefaultWebSessionManager;
 import org.zowe.apiml.gateway.filters.ForbidCharacterException;
 import org.zowe.apiml.gateway.filters.ForbidSlashException;
+import org.zowe.apiml.gateway.filters.ZaasInternalErrorException;
 import org.zowe.apiml.message.core.Message;
 import org.zowe.apiml.message.core.MessageService;
 import org.zowe.apiml.message.log.ApimlLogger;
@@ -163,7 +164,13 @@ public Mono<Void> handleStatusError(ServerWebExchange exchange, ResponseStatusEx
     @ExceptionHandler({ServiceNotAccessibleException.class, WebClientResponseException.ServiceUnavailable.class})
     public Mono<Void> handleServiceNotAccessibleException(ServerWebExchange exchange, Exception ex) {
         log.debug("A service is not available at the moment to finish request {}: {}", exchange.getRequest().getURI(), ex.getMessage());
-        return setBodyResponse(exchange, SC_SERVICE_UNAVAILABLE, "org.zowe.apiml.common.serviceUnavailable", exchange.getRequest().getURI());
+        return setBodyResponse(exchange, SC_SERVICE_UNAVAILABLE, "org.zowe.apiml.common.serviceUnavailable", exchange.getRequest().getPath());
+    }
+
+    @ExceptionHandler(ZaasInternalErrorException.class)
+    public Mono<Void> handleZaasInternalErrorException(ServerWebExchange exchange, ZaasInternalErrorException ex) {
+        log.debug("The ZAAS instance {} return internal server error for request {}: {}", ex.getInstanceId(), exchange.getRequest().getURI(), ex.getMessage());
+        return setBodyResponse(exchange, SC_INTERNAL_SERVER_ERROR, "org.zowe.apiml.gateway.zaas.internalServerError", ex.getInstanceId());
     }
 
 }

gateway-service/src/main/java/org/zowe/apiml/gateway/filters/AbstractAuthSchemeFactory.java

@@ -13,6 +13,7 @@
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.cloud.client.ServiceInstance;
 import org.springframework.cloud.gateway.filter.GatewayFilter;
@@ -35,12 +36,12 @@
 import java.net.HttpCookie;
 import java.security.cert.CertificateEncodingException;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
-import static org.apache.hc.core5.http.HttpStatus.SC_OK;
-import static org.apache.hc.core5.http.HttpStatus.SC_UNAUTHORIZED;
+import static org.apache.hc.core5.http.HttpStatus.*;
 import static org.zowe.apiml.constants.ApimlConstants.PAT_COOKIE_AUTH_NAME;
 import static org.zowe.apiml.constants.ApimlConstants.PAT_HEADER_NAME;
 import static org.zowe.apiml.gateway.x509.ForwardClientCertFilterFactory.CLIENT_CERT_HEADER;
@@ -111,9 +112,11 @@
  * private String token;
  * }
  */
+@Slf4j
 public abstract class AbstractAuthSchemeFactory<T extends AbstractAuthSchemeFactory.AbstractConfig, R, D> extends AbstractGatewayFilterFactory<T> {
 
     private static final String HEADER_SERVICE_ID = "X-Service-Id";
+    private static final String SERVICE_IS_UNAVAILABLE_MESSAGE = "There are no instance of ZAAS available";
 
     private static final String[] CERTIFICATE_HEADERS = {
         "X-Certificate-Public",
@@ -169,14 +172,35 @@ private Mono<List<ServiceInstance>> getZaasInstances() {
 
     private Mono<AuthorizationResponse<R>> requestWithHa(
         Iterator<ServiceInstance> serviceInstanceIterator,
-        Function<ServiceInstance, WebClient.RequestHeadersSpec<?>> requestCreator
+        Function<ServiceInstance, WebClient.RequestHeadersSpec<?>> requestCreator,
+        AtomicReference<Optional<Exception>> mostCriticalException // to be accessible and updatable in all lambdas below
     ) {
-        return requestCreator.apply(serviceInstanceIterator.next())
+        // selected instance of ZAAS to invoke
+        var zaasInstance = serviceInstanceIterator.next();
+
+        // this lambda creates a chain of call over all instances. It also remembers the most critical exception to
+        // be thrown in case all instances fail
+        Function<Exception, Mono<AuthorizationResponse<R>>> callNext = exception -> {
+            // select the most critical exception to remember (ZaasInternalErrorException is more important one)
+            exception = mostCriticalException.get().filter(ZaasInternalErrorException.class::isInstance).orElse(exception);
+            mostCriticalException.set(Optional.of(exception));
+
+            if (serviceInstanceIterator.hasNext()) {
+                return requestWithHa(serviceInstanceIterator, requestCreator, mostCriticalException);
+            } else {
+                return Mono.error(exception);
+            }
+        };
+
+        return requestCreator.apply(zaasInstance)
             .exchangeToMono(clientResp -> switch (clientResp.statusCode().value()) {
                 case SC_UNAUTHORIZED -> Mono.just(new AuthorizationResponse<>(clientResp.headers(), null));
                 case SC_OK -> clientResp.bodyToMono(getResponseClass()).map(b -> new AuthorizationResponse<>(clientResp.headers(), b));
-                default -> serviceInstanceIterator.hasNext() ? requestWithHa(serviceInstanceIterator, requestCreator) : Mono.just(new AuthorizationResponse<>(clientResp.headers(), null));
-            });
+                case SC_INTERNAL_SERVER_ERROR -> callNext.apply(new ZaasInternalErrorException(zaasInstance, "An internal exception occurred in ZAAS service. Check its configuration of instance " + zaasInstance.getInstanceId() + "."));
+                default -> callNext.apply(new ServiceNotAccessibleException(SERVICE_IS_UNAVAILABLE_MESSAGE));
+            })
+            .doOnError(t -> log.debug("Error on calling ZAAS service instance {}: {}", zaasInstance.getInstanceId(), t.getMessage()))
+            .onErrorResume(e -> callNext.apply(new ServiceNotAccessibleException(SERVICE_IS_UNAVAILABLE_MESSAGE)));
     }
 
     protected Mono<Void> invoke(
@@ -186,10 +210,12 @@ protected Mono<Void> invoke(
     ) {
         Iterator<ServiceInstance> i = robinRound.getIterator(serviceInstances);
         if (!i.hasNext()) {
-            throw new ServiceNotAccessibleException("There are no instance of ZAAS available");
+            throw new ServiceNotAccessibleException(SERVICE_IS_UNAVAILABLE_MESSAGE);
         }
 
-        return requestWithHa(i, requestCreator).switchIfEmpty(Mono.just(new AuthorizationResponse<>(null,null))).flatMap(responseProcessor);
+        return requestWithHa(i, requestCreator,  new AtomicReference<>(Optional.empty()))
+            .switchIfEmpty(Mono.just(new AuthorizationResponse<>(null,null)))
+            .flatMap(responseProcessor);
     }
 
     /**

gateway-service/src/main/java/org/zowe/apiml/gateway/filters/ZaasInternalErrorException.java

@@ -0,0 +1,26 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml.gateway.filters;
+
+import lombok.Getter;
+import org.springframework.cloud.client.ServiceInstance;
+
+public class ZaasInternalErrorException extends Exception {
+
+    @Getter
+    private final String instanceId;
+
+    public ZaasInternalErrorException(ServiceInstance zaasInstance, String message) {
+        super(message);
+        this.instanceId = zaasInstance.getInstanceId();
+    }
+
+}

gateway-service/src/main/resources/gateway-log-messages.yml

@@ -119,6 +119,13 @@ messages:
       reason: "Cannot connect to the Gateway service."
       action: "Make sure that the external Gateway service is running and the truststore of the both Gateways contain the corresponding certificate."
 
+    - key: org.zowe.apiml.gateway.zaas.internalServerError
+      number: ZWESG101
+      type: ERROR
+      text: "An internal exception occurred in ZAAS service %s."
+      reason: "ZAAS cannot process authentication required to finish the request."
+      action: "Make sure that the ZAAS is configured well and check all security requirements."
+
     - key: org.zowe.apiml.gateway.connectionsLimitApproached
       number: ZWESG429
       type: ERROR

gateway-service/src/test/java/org/zowe/apiml/gateway/acceptance/PassticketTest.java

@@ -28,8 +28,8 @@
 import java.util.Base64;
 
 import static io.restassured.RestAssured.given;
-import static org.apache.http.HttpStatus.SC_OK;
-import static org.apache.http.HttpStatus.SC_UNAUTHORIZED;
+import static org.apache.http.HttpStatus.*;
+import static org.hamcrest.Matchers.is;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
@@ -42,7 +42,6 @@ public class PassticketTest extends AcceptanceTestWithMockServices {
     private static final String JWT = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyIiwiaWF0IjoxNjcxNDYxNjIzLCJleHAiOjE2NzE0OTA0MjMsImlzcyI6IkFQSU1MIiwianRpIjoiYmFlMTkyZTYtYTYxMi00MThhLWI2ZGMtN2I0NWI5NzM4ODI3IiwiZG9tIjoiRHVtbXkgcHJvdmlkZXIifQ.Vt5UjJUlbmuzmmEIodAACtj_AOxlsWqkFrFyWh4_MQRRPCj_zMIwnzpqRN-NJvKtUg1zxOCzXv2ypYNsglrXc7cH9wU3leK1gjYxK7IJjn2SBEb0dUL5m7-h4tFq2zNhcGH2GOmTpE2gTQGSTvDIdja-TIj_lAvUtbkiorm1RqrNu2MGC0WfgOGiak3tj2tNJLv_Y1ZMxNjzyHgXBMuNPozQrd4Vtnew3x4yy85LrTYF7jJM3U-e3AD2yImftxwycQvbkjNb-lWadejTVH0MgHMr04wVdDd8Nq5q7yrZf7YPzhias8ehNbew5CHiKut9SseZ1sO2WwgfhpEfsN4okg";
     private static final String PASSTICKET = "ZOWE_DUMMY_PASS_TICKET";
 
-
     @Test
     void whenRequestingPassticketForAllowedAPPLID_thenTranslate() throws IOException {
         TicketResponse response = new TicketResponse();
@@ -51,15 +50,15 @@ void whenRequestingPassticketForAllowedAPPLID_thenTranslate() throws IOException
         response.setApplicationName("IZUDFLT");
         response.setTicket(PASSTICKET);
 
-        mockService("zaas").scope(MockService.Scope.CLASS)
+        mockService("zaas").scope(MockService.Scope.TEST)
             .addEndpoint("/zaas/scheme/ticket")
             .assertion(he -> assertEquals(SERVICE_ID, he.getRequestHeaders().getFirst("X-Service-Id")))
             .assertion(he -> assertEquals(COOKIE_NAME + "=" + JWT, he.getRequestHeaders().getFirst("Cookie")))
             .bodyJson(response)
             .and().start();
 
         String expectedAuthHeader = "Basic " + Base64.getEncoder().encodeToString((USER_ID + ":" + PASSTICKET).getBytes(StandardCharsets.UTF_8));
-        var mockService = mockService(SERVICE_ID)
+        var mockService = mockService(SERVICE_ID).scope(MockService.Scope.TEST)
             .authenticationScheme(AuthenticationScheme.HTTP_BASIC_PASSTICKET).applid("IZUDFLT")
             .addEndpoint("/" + SERVICE_ID + "/test")
             .assertion(he -> assertEquals(expectedAuthHeader, he.getRequestHeaders().getFirst(HttpHeaders.AUTHORIZATION)))
@@ -74,29 +73,69 @@ void whenRequestingPassticketForAllowedAPPLID_thenTranslate() throws IOException
         assertEquals(1, mockService.getEndpoint().getCounter());
     }
 
-
-    @ParameterizedTest
-    @ValueSource(ints = {400, 401, 403, 404, 405, 500})
-    void whenCannotGeneratePassticket_thenIgnoreTransformation(int responseCode) throws IOException {
+    @Test
+    void whenCredentialsAreMissingOrInvalid_thenIgnoreTransformation() throws IOException {
         mockService("zaas").scope(MockService.Scope.TEST)
             .addEndpoint("/zaas/scheme/ticket")
-            .responseCode(responseCode)
+            .responseCode(SC_UNAUTHORIZED)
             .and().start();
-        var mockService = mockService(SERVICE_ID).scope(MockService.Scope.TEST)
+        var service = mockService(SERVICE_ID).scope(MockService.Scope.TEST)
             .authenticationScheme(AuthenticationScheme.HTTP_BASIC_PASSTICKET).applid("IZUDFLT")
             .addEndpoint("/" + SERVICE_ID + "/test")
-            .responseCode(401)
-            .bodyJson(new ResponseDto("ok"))
+                .responseCode(SC_UNAUTHORIZED)
+                .bodyJson(new ResponseDto("ok"))
             .assertion(he -> assertFalse(he.getRequestHeaders().containsKey(HttpHeaders.AUTHORIZATION)))
             .and().start();
         given()
             .cookie(COOKIE_NAME, JWT)
-            .when()
+        .when()
             .get(basePath + "/" + SERVICE_ID + "/api/v1/test")
-            .then()
+        .then()
             .statusCode(Matchers.is(SC_UNAUTHORIZED))
             .body("status", Matchers.is("ok"));
-        assertEquals(1, mockService.getEndpoint().getCounter());
+        assertEquals(1, service.getEndpoint().getCounter());
+    }
+
+    @Test
+    void whenZaasIsMisconfigured_thenReturnError() throws IOException {
+        var zaas = mockService("zaas").scope(MockService.Scope.TEST)
+            .addEndpoint("/zaas/scheme/ticket")
+            .responseCode(SC_INTERNAL_SERVER_ERROR)
+            .and().start();
+        var service = mockService(SERVICE_ID).scope(MockService.Scope.TEST)
+            .authenticationScheme(AuthenticationScheme.HTTP_BASIC_PASSTICKET).applid("IZUDFLT")
+            .addEndpoint("/" + SERVICE_ID + "/test")
+            .and().start();
+        given()
+            .cookie(COOKIE_NAME, JWT)
+        .when()
+            .get(basePath + "/" + SERVICE_ID + "/api/v1/test")
+        .then()
+            .statusCode(Matchers.is(SC_INTERNAL_SERVER_ERROR))
+            .body("messages[0].messageKey", is("org.zowe.apiml.gateway.zaas.internalServerError"))
+            .body("messages[0].messageContent", is("An internal exception occurred in ZAAS service " + zaas.getInstanceId() + "."));
+        assertEquals(0, service.getEndpoint().getCounter());
+    }
+
+    @ParameterizedTest(name = "When ZAAS returns {0} the Gateway response with 503")
+    @ValueSource(ints = {400, 403, 404, 405})
+    void whenCannotGeneratePassticket_thenReturn503(int responseCode) throws IOException {
+        mockService("zaas").scope(MockService.Scope.TEST)
+            .addEndpoint("/zaas/scheme/ticket")
+            .responseCode(responseCode)
+            .and().start();
+        var service = mockService(SERVICE_ID).scope(MockService.Scope.TEST)
+            .authenticationScheme(AuthenticationScheme.HTTP_BASIC_PASSTICKET).applid("IZUDFLT")
+            .addEndpoint("/" + SERVICE_ID + "/test")
+            .and().start();
+        given()
+            .cookie(COOKIE_NAME, JWT)
+        .when()
+            .get(basePath + "/" + SERVICE_ID + "/api/v1/test")
+        .then()
+            .statusCode(Matchers.is(SC_SERVICE_UNAVAILABLE))
+            .body("messages[0].messageKey", is("org.zowe.apiml.common.serviceUnavailable"));
+        assertEquals(0, service.getEndpoint().getCounter());
     }
 
     @Data
@@ -106,7 +145,5 @@ static class ResponseDto {
         private String status;
 
     }
-}
-
-
 
+}