[Backport 8.x] Using matching docker container server version in unit tests

java-client/src/test/java/co/elastic/clients/elasticsearch/ElasticsearchTestServer.java

@@ -29,6 +29,7 @@
 import co.elastic.clients.transport.Version;
 import co.elastic.clients.transport.endpoints.DelegatingJsonEndpoint;
 import co.elastic.clients.transport.rest_client.RestClientTransport;
+import org.apache.commons.io.FileUtils;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
@@ -40,12 +41,15 @@
 import org.testcontainers.utility.DockerImageName;
 
 import javax.net.ssl.SSLContext;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.Base64;
 
 public class ElasticsearchTestServer implements AutoCloseable {
@@ -58,6 +62,7 @@ public class ElasticsearchTestServer implements AutoCloseable {
     private ElasticsearchClient client;
 
     private static ElasticsearchTestServer global;
+    private static final String artifactsApiUrl = "https://artifacts-api.elastic.co/v1/versions/";
 
     public static synchronized ElasticsearchTestServer global() {
         if (global == null) {
@@ -112,14 +117,70 @@ AuthScope.ANY, new UsernamePasswordCredentials("elastic", "changeme")
         client = new ElasticsearchClient(transport);
     }
 
+    private Version selectLatestVersion(Version version, String info) {
+        if (info.contains(version.toString())) {
+            return version;
+        }
+        // if no version X.Y.0 was found, we give up
+        if (version.maintenance() == 0) {
+            throw new RuntimeException("Elasticsearch server container version: " + version + " not yet " +
+                "available");
+        }
+        return selectLatestVersion(new Version(version.major(), version.minor(), version.maintenance() - 1,
+            false), info);
+    }
+
+    private String fetchAndWriteVersionInfo(File file) throws IOException {
+        String versionInfo = IOUtils.toString(new URL(artifactsApiUrl), StandardCharsets.UTF_8);
+        FileUtils.writeStringToFile(file, versionInfo, StandardCharsets.UTF_8);
+        return versionInfo;
+    }
+
+    private Version getLatestAvailableServerVersion(Version version) {
+        try {
+            // check if there's cached information
+            ClassLoader classLoader = getClass().getClassLoader();
+            URL location = classLoader.getResource("./co/elastic/clients/version.json");
+
+            // writing the info on file before returning
+            if (location == null) {
+                File file = new File(classLoader.getResource("./co/elastic/clients").getFile() + "/version" +
+                    ".json");
+                String versionInfo = fetchAndWriteVersionInfo(file);
+                return selectLatestVersion(version, versionInfo);
+            }
+
+            File file = new File(location.getFile());
+
+            // info file was found, but it's expired
+            if (Instant.ofEpochMilli(file.lastModified()).isBefore(Instant.now().minus(24,
+                ChronoUnit.HOURS))) {
+                String versionInfo = fetchAndWriteVersionInfo(file);
+                return selectLatestVersion(version, versionInfo);
+            }
+
+            // info file exists and it has new info
+            String versionInfo = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
+            return selectLatestVersion(version, versionInfo);
+
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public synchronized ElasticsearchTestServer start() {
         if (this.client != null) {
             return this;
         }
 
-        Version version = Version.VERSION.major() < 8 ? new Version(7,17,5,false) : new Version(8,12,0,false);
+        Version version = getLatestAvailableServerVersion(Version.VERSION);
+
+        // using specific stable version for tests with plugins
+        if (plugins.length > 0) {
+            version = Version.VERSION.major() < 8 ? new Version(7, 17, 25, false) : new Version(8, 16, 0,
+                false);
+        }
 
-        // Note we could use version.major() + "." + version.minor() + "-SNAPSHOT" but plugins won't install on a snapshot version
         String esImage = "docker.elastic.co/elasticsearch/elasticsearch:" + version;
 
         DockerImageName image;
@@ -166,10 +227,11 @@ public static <Req> JsonData getJsonResponse(ElasticsearchClient client, Req req
 
         try {
             @SuppressWarnings("unchecked")
-            JsonEndpoint<Req, JsonData, ErrorResponse> endpoint0 = (JsonEndpoint<Req, JsonData, ErrorResponse>) request.getClass()
+            JsonEndpoint<Req, JsonData, ErrorResponse> endpoint0 = (JsonEndpoint<Req, JsonData,
+                ErrorResponse>) request.getClass()
                 .getDeclaredField("_ENDPOINT").get(null);
             endpoint = endpoint0;
-        } catch (IllegalAccessException|NoSuchFieldException e) {
+        } catch (IllegalAccessException | NoSuchFieldException e) {
             throw new RuntimeException(e);
         }