diff --git a/docs/_docs/analysis-types/outdated-components.md b/docs/_docs/analysis-types/outdated-components.md index 1236bfb474..5ec1f4873a 100644 --- a/docs/_docs/analysis-types/outdated-components.md +++ b/docs/_docs/analysis-types/outdated-components.md @@ -29,4 +29,5 @@ components ecosystem. Refer to [Repositories]({{ site.baseurl }}{% link _docs/da further information. ### Stable releases -In some repositories, such as NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. For Maven repositories Dependency Track tries find the latest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the latest unstable version is reported. +In some repositories, for example NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. In NPM as wel as Maven repositories the latest version does not need to be the highest version. It's just the latest published to the repository. + diff --git a/docs/_docs/datasources/repositories.md b/docs/_docs/datasources/repositories.md index b8644bc5b4..6928de3d9f 100644 --- a/docs/_docs/datasources/repositories.md +++ b/docs/_docs/datasources/repositories.md @@ -58,3 +58,13 @@ leveraging repositories to identify outdated components. Refer to [Datasource Routing]({{ site.baseurl }}{% link _docs/datasources/routing.md %}) for information on Package URL and the various ways it is used throughout Dependency-Track. + +#### Highest stable release +Dependency Track identifies outdated components by looking for newer versions of the component. Preferably this should be +a higher version, but usualy repositories report the latest version of a component which is the last updated version. Also +some repositories report unstable versions as the latest version. + +Dependency Track tries find the highest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the highest unstable version is reported. This feature is currently suported for the following repositories: + * Maven + +For all other repositories the latest version as reported by the repository is used. \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java index bca39977dd..8c5f29c8d7 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java @@ -18,15 +18,19 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; +import java.util.List; + import org.apache.commons.lang3.StringUtils; +import org.apache.maven.artifact.versioning.ComparableVersion; import org.dependencytrack.model.Component; import org.dependencytrack.notification.NotificationConstants; import org.dependencytrack.notification.NotificationGroup; import org.dependencytrack.notification.NotificationScope; +import alpine.common.logging.Logger; +import alpine.notification.Notification; +import alpine.notification.NotificationLevel; + /** * Base abstract class that all IMetaAnalyzer implementations should likely extend. * @@ -35,6 +39,8 @@ */ public abstract class AbstractMetaAnalyzer implements IMetaAnalyzer { + protected static final String UNSTABLE_VERSIONS_PATTERN = "(?i).*[-_\\.](atlassian|preview|snapshot|a|alpha|b|beta|rc|m|ea)[-_\\.]?\\d*"; // ignore case + protected String baseUrl; protected String username; @@ -84,4 +90,60 @@ protected void handleRequestException(final Logger logger, final Exception e) { ); } + + + /** + * Parse two version strings and return the one containing the highest version + * + * @param v1string first version to compare + * @param v2string second version to compare + * @return the highest of two versions as string value + */ + protected static String highestVersion(String v1string, String v2string) { + if (v1string == null) { + return v2string; + } else if (v2string == null) { + return v1string; + } else { + ComparableVersion v1 = new ComparableVersion(v1string); + ComparableVersion v2 = new ComparableVersion(v2string); + return v1.compareTo(v2) > 0 ? v1string : v2string; + } + } + + /** + * Determine wether a version string denotes a stable version + * @param version + * @return true if the version string denotes a stable version + */ + protected static boolean isStableVersion(String version) { + return !version.matches(UNSTABLE_VERSIONS_PATTERN); + } + + /** + * Get the highest version from a list of version strings + * + * @param versions list of version strings + * @return the highest version in the list + */ + protected static String getHighestVersion(List versions) { + String highestStableVersion = null; + if (!versions.isEmpty()) { + highestStableVersion = versions.stream().reduce(null, AbstractMetaAnalyzer::highestVersion); + } + return highestStableVersion; + } + + /** + * Get the highest stable version from a list of version strings + * + * @param versions list of version strings + * @return the highest version in the list + */ + protected static String getHighestStableVersion(List versions) { + // collect stable versions + List stableVersions = versions.stream().filter(AbstractMetaAnalyzer::isStableVersion).toList(); + return getHighestVersion(stableVersions); + } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java index 304671a819..1d36630786 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java @@ -35,7 +35,6 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; -import org.apache.maven.artifact.versioning.ComparableVersion; import org.dependencytrack.common.HttpClientPool; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; @@ -61,31 +60,10 @@ public class MavenMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(MavenMetaAnalyzer.class); private static final String DEFAULT_BASE_URL = "https://repo1.maven.org/maven2"; - private static final String REPO_METADATA_URL = "/%s/maven-metadata.xml"; - protected static final String UNSTABLE_VERSIONS_PATTERN = "(?i).*[-_\\.](atlassian|preview|snapshot|a|alpha|b|beta|rc|m|ea)[-_\\.]?[0-9]*"; // ignore case MavenMetaAnalyzer() { this.baseUrl = DEFAULT_BASE_URL; } - - /** - * Parse two version strings and return the one containing the highest version - * - * @param v1string first version to compare - * @param v2string second version to compare - * @return the highest of two versions as string value - */ - private String highestVersion(String v1string, String v2string) { - if (v1string == null) { - return v2string; - } else if (v2string == null) { - return v1string; - } else { - ComparableVersion v1 = new ComparableVersion(v1string); - ComparableVersion v2 = new ComparableVersion(v2string); - return v1.compareTo(v2) > 0 ? v1string : v2string; - } - } /** * {@inheritDoc} @@ -107,8 +85,8 @@ public RepositoryType supportedRepositoryType() { public MetaModel analyze(final Component component) { final MetaModel meta = new MetaModel(component); if (component.getPurl() != null) { - final String mavenGavUrl = component.getPurl().getNamespace().replaceAll("\\.", "/") + "/" + component.getPurl().getName(); - final String url = String.format(baseUrl + REPO_METADATA_URL, mavenGavUrl); + final String mavenGavUrl = component.getPurl().getNamespace().replace(".", "/") + "/" + component.getPurl().getName(); + final String url = String.format("%s%s", baseUrl, String.format("/%s/maven-metadata.xml", mavenGavUrl)); try { final HttpUriRequest request = new HttpGet(url); @@ -122,59 +100,7 @@ public MetaModel analyze(final Component component) { final HttpEntity entity = response.getEntity(); if (entity != null) { try (InputStream in = entity.getContent()) { - final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in); - final XPathFactory xpathFactory = XPathFactory.newInstance(); - final XPath xpath = xpathFactory.newXPath(); - - // release: What the latest version in the directory is, of the releases only - final XPathExpression releaseExpression = xpath.compile("/metadata/versioning/release"); - final String release = (String) releaseExpression.evaluate(document, XPathConstants.STRING); - final String stableRelease = release.matches(UNSTABLE_VERSIONS_PATTERN) ? null : release; - - // latest: What the latest version in the directory is, including snapshots - final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest"); - final String latest = (String) latestExpression.evaluate(document, XPathConstants.STRING); - // No need to determine stableLatest as it wil always be null or release: - // final String stableLatest = !latest.matches(UNSTABLE_VERSIONS_PATTERN) ? null : latest; - - final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated"); - final String lastUpdated = (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING); - - if (stableRelease != null) { - meta.setLatestVersion(stableRelease); - if (lastUpdated != null && stableRelease.equals(latest)) { - // lastUpdated reflects the timestamp when latest was updated, so it's only valid when stableRelease == latest - meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); - } - } else { - // Determine latest stable version from list of versions - // versions/version*: (Many) Versions available of the artifact (both releases and snapshots) - final XPathExpression versionsExpression = xpath.compile("/metadata/versioning/versions/*"); - final NodeList versionsList = (NodeList) versionsExpression.evaluate(document, XPathConstants.NODESET); - - // collect stable versions - List stableVersions = new ArrayList<>(); - for (int n = 0; n < versionsList.getLength(); n++) { - Node versionNode = versionsList.item(n); - String version = versionNode.getFirstChild().getNodeValue().trim(); - if (!version.matches(UNSTABLE_VERSIONS_PATTERN)) { - stableVersions.add(version); - } - } - - // find latest stable version - String latestStableVersion = null; - if (stableVersions.size() > 0) { - latestStableVersion = stableVersions.stream().reduce(null, (highest, current) -> highestVersion(highest, current)); - } - // use latestStableVersion, or else latest unstable release (e.g. alpha, milestone) or else latest snapshot - String latestVersion = latestStableVersion != null ? latestStableVersion: (release != null ? release: latest); - meta.setLatestVersion(latestVersion); - if (lastUpdated != null && latestVersion.equals(latest)) { - // lastUpdated reflects the timestamp when latest was updated, so it's only valid when latestVersion == latest - meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); - } - } + analyzeContent(meta, in); } } } else { @@ -187,4 +113,76 @@ public MetaModel analyze(final Component component) { } return meta; } + + /** + * The maven-metadata.xml files are not updated by Nexus during deployment, they are updated by Maven. It downloads the file, updates it, and then redeploys it. + * + * Maven will update the "release" field only during the following scenarios: + * + * 1. Maven 2.x deploys using -DupdateReleaseInfo=true + * 2. Maven 3.x deploys a non-snapshot artifact + * + * The "latest" field is only intended for plugin resolution, and is only set upon deployment of a maven-plugin artifact, both for Maven 2.x and 3.x regardless whether a release or snapshot gets deployed. + * + * Also, Maven will update these fields with whatever version it is currently deploying, so "latest" and "release" will not necessarily correspond to the highest version number. + * + * https://support.sonatype.com/hc/en-us/articles/213464638-Why-are-the-latest-and-release-tags-in-maven-metadata-xml-not-being-updated-after-deploying-artifacts- + */ + private void analyzeContent(final MetaModel meta, InputStream in) + throws SAXException, IOException, ParserConfigurationException, XPathExpressionException { + final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in); + final XPathFactory xpathFactory = XPathFactory.newInstance(); + final XPath xpath = xpathFactory.newXPath(); + + // latest: What the latest version in the directory is, including snapshots + final String latest = getLatestVersionFromMetadata(document, xpath); + // When the metadata was last updated + final String lastUpdated = getLastUpdatedFromMetadata(document, xpath); + // versions/version*: (Many) Versions available of the artifact (both releases and snapshots) + final NodeList versionsList = getVersionsFromMetadata(document, xpath); + + // latest and release might not be the highest version in case of a hotfix on an older release! + + // find highest stable or unstable version from list of versions + List versions = getVersions(versionsList); + // find highest stable version from list of versions + String highestStableOrUnstableVersion = AbstractMetaAnalyzer.getHighestVersion(versions); + + // find highest stable version + String highestStableVersion = getHighestStableVersion(versions); + + // use highestStableVersion, or else latest unstable release (e.g. alpha, milestone) or else latest snapshot + String wantedVersion = highestStableVersion != null ? highestStableVersion: highestStableOrUnstableVersion; + meta.setLatestVersion(wantedVersion); + if (lastUpdated != null && wantedVersion != null && wantedVersion.equals(latest)) { + // lastUpdated reflects the timestamp when latest was updated, so it's only valid when wantedVersion == latest + meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); + } + } + + private NodeList getVersionsFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression versionsExpression = xpath.compile("/metadata/versioning/versions/*"); + return (NodeList) versionsExpression.evaluate(document, XPathConstants.NODESET); + } + + private String getLastUpdatedFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated"); + return (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING); + } + + private String getLatestVersionFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest"); + return (String) latestExpression.evaluate(document, XPathConstants.STRING); + } + + private List getVersions(final NodeList versionsList) { + List versions = new ArrayList<>(); + for (int n = 0; n < versionsList.getLength(); n++) { + Node versionNode = versionsList.item(n); + String version = versionNode.getFirstChild().getNodeValue().trim(); + versions.add(version); + } + return versions; + } + } diff --git a/src/test/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzerTest.java new file mode 100644 index 0000000000..f79f155bd5 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzerTest.java @@ -0,0 +1,87 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.tasks.repositories; + +import java.util.Arrays; + +import org.junit.Assert; +import org.junit.Test; + +public class AbstractMetaAnalyzerTest { + + @Test + public void testHighestVersion() { + Assert.assertNull(AbstractMetaAnalyzer.highestVersion(null, null)); + Assert.assertEquals("0.0.1", AbstractMetaAnalyzer.highestVersion("0.0.1", null)); + Assert.assertEquals("0.0.1", AbstractMetaAnalyzer.highestVersion(null, "0.0.1")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("2-SNAPSHOT", "2")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("2-RC1", "2")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("2-alpha", "2")); + Assert.assertEquals("2.0.0", AbstractMetaAnalyzer.highestVersion("1.9.9.9", "2.0.0")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("1.9", "2")); + Assert.assertEquals("0.1", AbstractMetaAnalyzer.highestVersion("0.1", "0.0.1")); + // N.B. we would like to see 2 being higher as 2-a, but since we're only interested in stable versions it doesn't maater + Assert.assertEquals("2-a", AbstractMetaAnalyzer.highestVersion("2-a", "2")); + } + + @Test + public void testIsStableVersion() { + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("version")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1.0")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1.0.0")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1.0.0")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1a")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1-a")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1-alpha")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1-b")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1.1-snapshot")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1.2.3-rc1")); + } + + @Test + public void testGetHighestVersion() { + Assert.assertNull(AbstractMetaAnalyzer.getHighestVersion(Arrays.asList())); + Assert.assertEquals("1", AbstractMetaAnalyzer.getHighestVersion(Arrays.asList("1"))); + Assert.assertEquals("2-snapshot", AbstractMetaAnalyzer.getHighestVersion(Arrays.asList("1", "2-snapshot"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestVersion(Arrays.asList("1", "2"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestVersion(Arrays.asList("1", "2", "2-snapshot"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestVersion(Arrays.asList("1", "2", "2-alpha"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestVersion(Arrays.asList("1", "2", "2-beta"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestVersion(Arrays.asList("1", "2", "2-rc1"))); + } + + @Test + public void testGetHighestStableVersion() { + Assert.assertNull(AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList())); + Assert.assertEquals("1", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "0.9.9"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2-snapshot"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2-rc1"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2-alpha"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2-m1"))); + Assert.assertEquals("2.0", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1.2.3", "2.0"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2", "2-snapshot"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2", "2-a"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2", "2-alpha"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2", "2-beta"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1", "2", "2-rc1"))); + Assert.assertNull(AbstractMetaAnalyzer.getHighestStableVersion(Arrays.asList("1-a", "2-snapshot", "2-rc1"))); + } +} diff --git a/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java index dd997d7d51..6022779de0 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java @@ -80,4 +80,5 @@ public void testAnalyzerForScalaComponent() throws Exception { // publishedTimestamp might be set or not } } + }