Restrict latest versions to stable highest releases only for all default repositories

docs/_docs/analysis-types/outdated-components.md

@@ -27,3 +27,9 @@ from various repositories. Dependency-Track relies on Package URL (PURL) to iden
 to, the metadata about the component, and uses that data to query the various repositories capable of supporting the 
 components ecosystem. Refer to [Repositories]({{ site.baseurl }}{% link _docs/datasources/repositories.md %}) for 
 further information.
+
+### Stable releases
+In some repositories, for example NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a stable release or an unstable version. In NPM as well as Maven repositories, the latest version does not need to be the highest version. It's just the latest published to the repository.
+
+For some repositories, Dependency-Track tries to find the highest stable release instead of just the latest version. Refer to [Repositories]({{ site.baseurl }}{% link _docs/datasources/repositories.md %}) for further information.
+

docs/_docs/datasources/repositories.md

@@ -59,3 +59,17 @@ leveraging repositories to identify outdated components.
 
 Refer to [Datasource Routing]({{ site.baseurl }}{% link _docs/datasources/routing.md %})
 for information on Package URL and the various ways it is used throughout Dependency-Track.
+
+#### Highest stable release
+Dependency-Track identifies outdated components by looking for newer versions of the component. Preferably this should be a higher version, but usualy repositories report the newest version of a component which might not be the highest version. Also some repositories report unstable versions as the latest version.
+
+Dependency-Track tries find the highest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the highest unstable version is reported. This feature is suported for all default repositories:
+  * Cargo
+  * Composer
+  * Gem
+  * Go
+  * Hex
+  * Maven
+  * NPM
+  * NuGet
+  * PyPi

pom.xml

@@ -227,6 +227,12 @@
             <artifactId>lucene-sandbox</artifactId>
             <version>${lib.lucene.version}</version>
         </dependency>
+        <!-- Semver4j-->
+        <dependency>
+            <groupId>com.vdurmont</groupId>
+            <artifactId>semver4j</artifactId>
+            <version>3.1.0</version>
+        </dependency>
         <!-- Pebble templating engine -->
         <dependency>
             <groupId>io.pebbletemplates</groupId>

src/main/java/org/dependencytrack/policy/VersionPolicyEvaluator.java

@@ -18,14 +18,13 @@
  */
 package org.dependencytrack.policy;
 
-import alpine.common.logging.Logger;
+import java.util.ArrayList;
+import java.util.List;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.Policy;
 import org.dependencytrack.model.PolicyCondition;
 import org.dependencytrack.util.ComponentVersion;
-
-import java.util.ArrayList;
-import java.util.List;
+import alpine.common.logging.Logger;
 
 /**
  * Evaluates a components version against a policy.
@@ -56,10 +55,6 @@ public List<PolicyConditionViolation> evaluate(final Policy policy, final Compon
             LOGGER.debug("Evaluating component (" + component.getUuid() + ") against policy condition (" + condition.getUuid() + ")");
 
             final var conditionVersion = new ComponentVersion(condition.getValue());
-            if (conditionVersion.getVersionParts().isEmpty()) {
-                LOGGER.warn("Unable to parse version (" + condition.getValue() + " provided by condition");
-                continue;
-            }
 
             if (matches(componentVersion, conditionVersion, condition.getOperator())) {
                 violations.add(new PolicyConditionViolation(condition, component));

src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java

@@ -18,23 +18,23 @@
  */
 package org.dependencytrack.tasks.repositories;
 
-import alpine.common.logging.Logger;
-import alpine.notification.Notification;
-import alpine.notification.NotificationLevel;
+import java.io.IOException;
+import java.net.URISyntaxException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.utils.URIBuilder;
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.dependencytrack.common.HttpClientPool;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.notification.NotificationConstants;
 import org.dependencytrack.notification.NotificationGroup;
 import org.dependencytrack.notification.NotificationScope;
 import org.dependencytrack.util.HttpUtil;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
+import alpine.common.logging.Logger;
+import alpine.notification.Notification;
+import alpine.notification.NotificationLevel;
 
 /**
  * Base abstract class that all IMetaAnalyzer implementations should likely extend.
@@ -49,6 +49,7 @@ public abstract class AbstractMetaAnalyzer implements IMetaAnalyzer {
     protected String username;
 
     protected String password;
+
     /**
      * {@inheritDoc}
      */
@@ -108,4 +109,29 @@ protected CloseableHttpResponse processHttpRequest(String url) throws IOExceptio
         }
     }
 
+    /**
+     * Parse two version strings and return the one containing the highest version
+     *
+     * @param v1string first version to compare
+     * @param v2string second version to compare
+     * @return the highest of two versions as string value
+     */
+    public static String highestVersion(String v1string, String v2string) {
+        if (v1string == null) {
+            return v2string;
+        } else if (v2string == null) {
+            return v1string;
+        } else {
+            ComparableVersion v1 = new ComparableVersion(stripLeadingV(v1string));
+            ComparableVersion v2 = new ComparableVersion(stripLeadingV(v2string));
+            return v1.compareTo(v2) > 0 ? v1string : v2string;
+        }
+    }
+
+    protected static String stripLeadingV(String s) {
+        return s.startsWith("v")
+                ? s.substring(1)
+                : s;
+    }
+
 }

src/main/java/org/dependencytrack/tasks/repositories/CargoMetaAnalyzer.java

@@ -18,20 +18,19 @@
  */
 package org.dependencytrack.tasks.repositories;
 
-import alpine.common.logging.Logger;
-import com.github.packageurl.PackageURL;
-import org.dependencytrack.exception.MetaAnalyzerException;
-import org.json.JSONArray;
-import org.json.JSONObject;
+import java.io.IOException;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.util.EntityUtils;
+import org.dependencytrack.exception.MetaAnalyzerException;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.RepositoryType;
 import org.dependencytrack.util.DateUtil;
-
-import java.io.IOException;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import com.github.packageurl.PackageURL;
+import alpine.common.logging.Logger;
 
 /**
  * An IMetaAnalyzer implementation that supports Cargo via crates.io compatible repos
@@ -74,28 +73,11 @@ public MetaModel analyze(final Component component) {
                 if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                     final HttpEntity entity = response.getEntity();
                     if (entity != null) {
-                        String responseString = EntityUtils.toString(entity);
-                        var jsonObject = new JSONObject(responseString);
+                        final String responseString = EntityUtils.toString(entity);
+                        final JSONObject jsonObject = new JSONObject(responseString);
                         final JSONObject crate = jsonObject.optJSONObject("crate");
-                        if (crate != null) {
-                            final String latest = crate.getString("newest_version");
-                            meta.setLatestVersion(latest);
-                        }
                         final JSONArray versions = jsonObject.optJSONArray("versions");
-                        if (versions != null) {
-                            for (int i = 0; i < versions.length(); i++) {
-                                final JSONObject version = versions.getJSONObject(i);
-                                final String versionString = version.optString("num");
-                                if (meta.getLatestVersion() != null && meta.getLatestVersion().equals(versionString)) {
-                                    final String publishedTimestamp = version.optString("created_at");
-                                    try {
-                                        meta.setPublishedTimestamp(DateUtil.fromISO8601(publishedTimestamp));
-                                    } catch (IllegalArgumentException e) {
-                                        LOGGER.warn("An error occurred while parsing published time", e);
-                                    }
-                                }
-                            }
-                        }
+                        analyzeCrate(meta, crate, versions);
                     }
                 } else {
                     handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component);
@@ -108,4 +90,26 @@ public MetaModel analyze(final Component component) {
         }
         return meta;
     }
+
+    private void analyzeCrate(final MetaModel meta, final JSONObject crate, final JSONArray versions) {
+        if (crate != null) {
+            // Cargo has a highest stable version: https://github.com/rust-lang/crates.io/pull/3163
+            final String latest = crate.getString("max_stable_version");
+            meta.setLatestVersion(latest);
+        }
+        if (versions != null) {
+            for (int i=0; i<versions.length(); i++) {
+                final JSONObject version =  versions.getJSONObject(i);
+                final String versionString = version.optString("num");
+                if (meta.getLatestVersion() != null && meta.getLatestVersion().equals(versionString)) {
+                    final String publishedTimestamp = version.optString("created_at");
+                    try {
+                        meta.setPublishedTimestamp(DateUtil.fromISO8601(publishedTimestamp));
+                    } catch (IllegalArgumentException  e) {
+                        LOGGER.warn("An error occurred while parsing published time", e);
+                    }
+                }
+            }
+        }
+    }
 }

src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java

@@ -18,21 +18,24 @@
  */
 package org.dependencytrack.tasks.repositories;
 
-import alpine.common.logging.Logger;
-import com.github.packageurl.PackageURL;
-import org.dependencytrack.exception.MetaAnalyzerException;
-import org.json.JSONObject;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.util.EntityUtils;
-import org.apache.maven.artifact.versioning.ComparableVersion;
+import org.dependencytrack.exception.MetaAnalyzerException;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.RepositoryType;
-
-import java.io.IOException;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
+import org.dependencytrack.util.ComponentVersion;
+import org.json.JSONObject;
+import com.github.packageurl.PackageURL;
+import alpine.common.logging.Logger;
 
 /**
  * An IMetaAnalyzer implementation that supports Composer.
@@ -103,34 +106,23 @@ public MetaModel analyze(final Component component) {
             }
             final JSONObject composerPackage = responsePackages.getJSONObject(expectedResponsePackage);
 
-            final ComparableVersion latestVersion = new ComparableVersion(stripLeadingV(component.getPurl().getVersion()));
-            final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
-
-            composerPackage.names().forEach(key_ -> {
-                String key = (String) key_;
-                if (key.startsWith("dev-") || key.endsWith("-dev")) {
+            // get list of versions and published timestamps
+            Map<String, String> versions = new HashMap<>();
+            composerPackage.names().forEach(keyObject -> {
+                String key = (String) keyObject;
+                if (!key.startsWith("dev-") && !key.endsWith("-dev")) {
                     // dev versions are excluded, since they are not pinned but a VCS-branch.
-                    return;
-                }
-
-                final String version_normalized = composerPackage.getJSONObject(key).getString("version_normalized");
-                ComparableVersion currentComparableVersion = new ComparableVersion(version_normalized);
-                if (currentComparableVersion.compareTo(latestVersion) < 0) {
-                    // smaller version can be skipped
-                    return;
+                    final String version = composerPackage.getJSONObject(key).getString("version");
+                    final String published = composerPackage.getJSONObject(key).getString("time");
+                    versions.put(version, published);
                 }
+            });
+            final String highestVersion = ComponentVersion.findHighestVersion(new ArrayList<>(versions.keySet()));
+            meta.setLatestVersion(highestVersion);
 
-                final String version = composerPackage.getJSONObject(key).getString("version");
-                latestVersion.parseVersion(stripLeadingV(version_normalized));
-                meta.setLatestVersion(version);
+            final String published = versions.get(highestVersion);
+            meta.setPublishedTimestamp(getPublisedTimestamp(published));
 
-                final String published = composerPackage.getJSONObject(key).getString("time");
-                try {
-                    meta.setPublishedTimestamp(dateFormat.parse(published));
-                } catch (ParseException e) {
-                    LOGGER.warn("An error occurred while parsing upload time", e);
-                }
-            });
         } catch (IOException ex) {
             handleRequestException(LOGGER, ex);
         } catch (Exception ex) {
@@ -140,9 +132,15 @@ public MetaModel analyze(final Component component) {
         return meta;
     }
 
-    private static String stripLeadingV(String s) {
-        return s.startsWith("v")
-                ? s.substring(1)
-                : s;
+    private Date getPublisedTimestamp(final String published) {
+        Date publishedTimestamp = null;
+        try {
+            final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
+            publishedTimestamp = dateFormat.parse(published);
+        } catch (ParseException e) {
+            LOGGER.warn("An error occurred while parsing upload time", e);
+        }
+        return publishedTimestamp;
     }
+
 }