Skip to content

Commit

Permalink
Restrict latest versions to stable releases only for maven repositories
Browse files Browse the repository at this point in the history 
Signed-off-by: Walter de Boer <[email protected]>
  • Loading branch information
Walter de Boer committed Feb 18, 2023
1 parent 22791bf commit cbfe8c1
Show file tree
Hide file tree
Showing 6 changed files with 240 additions and 81 deletions.
3 changes: 2 additions & 1 deletion docs/_docs/analysis-types/outdated-components.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,5 @@ components ecosystem. Refer to [Repositories]({{ site.baseurl }}{% link _docs/da
further information.

### Stable releases
In some repositories, such as NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. For Maven repositories Dependency Track tries find the latest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the latest unstable version is reported.
In some repositories, for example NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. In NPM as wel as Maven repositories the latest version does not need to be the highest version. It's just the latest published to the repository.

10 changes: 10 additions & 0 deletions docs/_docs/datasources/repositories.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,13 @@ leveraging repositories to identify outdated components.

Refer to [Datasource Routing]({{ site.baseurl }}{% link _docs/datasources/routing.md %})
for information on Package URL and the various ways it is used throughout Dependency-Track.

#### Highest stable release
Dependency Track identifies outdated components by looking for newer versions of the component. Preferably this should be
a higher version, but usualy repositories report the latest version of a component which is the last updated version. Also
some repositories report unstable versions as the latest version.

Dependency Track tries find the highest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the highest unstable version is reported. This feature is currently suported for the following repositories:
* Maven

For all other repositories the latest version as reported by the repository is used.
68 changes: 65 additions & 3 deletions src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,19 @@
*/
package org.dependencytrack.tasks.repositories;

import alpine.common.logging.Logger;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import org.apache.maven.artifact.versioning.ComparableVersion;
import org.dependencytrack.model.Component;
import org.dependencytrack.notification.NotificationConstants;
import org.dependencytrack.notification.NotificationGroup;
import org.dependencytrack.notification.NotificationScope;

import alpine.common.logging.Logger;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;

/**
* Base abstract class that all IMetaAnalyzer implementations should likely extend.
*
Expand All @@ -35,6 +39,8 @@
*/
public abstract class AbstractMetaAnalyzer implements IMetaAnalyzer {

protected static final String UNSTABLE_VERSIONS_PATTERN = "(?i).*[-_\\.](atlassian|preview|snapshot|a|alpha|b|beta|rc|m|ea)[-_\\.]?\\d*"; // ignore case

protected String baseUrl;

protected String username;
Expand Down Expand Up @@ -84,4 +90,60 @@ protected void handleRequestException(final Logger logger, final Exception e) {
);
}



/**
* Parse two version strings and return the one containing the highest version
*
* @param v1string first version to compare
* @param v2string second version to compare
* @return the highest of two versions as string value
*/
protected static String highestVersion(String v1string, String v2string) {
if (v1string == null) {
return v2string;
} else if (v2string == null) {
return v1string;
} else {
ComparableVersion v1 = new ComparableVersion(v1string);
ComparableVersion v2 = new ComparableVersion(v2string);
return v1.compareTo(v2) > 0 ? v1string : v2string;
}
}

/**
* Determine wether a version string denotes a stable version
* @param version
* @return true if the version string denotes a stable version
*/
protected static boolean isStableVersion(String version) {
return !version.matches(UNSTABLE_VERSIONS_PATTERN);
}

/**
* Get the highest version from a list of version strings
*
* @param versions list of version strings
* @return the highest version in the list
*/
protected static String getHighestVersion(List<String> versions) {
String highestStableVersion = null;
if (!versions.isEmpty()) {
highestStableVersion = versions.stream().reduce(null, AbstractMetaAnalyzer::highestVersion);
}
return highestStableVersion;
}

/**
* Get the highest stable version from a list of version strings
*
* @param versions list of version strings
* @return the highest version in the list
*/
protected static String getHighestStableVersion(List<String> versions) {
// collect stable versions
List<String> stableVersions = versions.stream().filter(AbstractMetaAnalyzer::isStableVersion).toList();
return getHighestVersion(stableVersions);
}

}
152 changes: 75 additions & 77 deletions src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.maven.artifact.versioning.ComparableVersion;
import org.dependencytrack.common.HttpClientPool;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;
Expand All @@ -61,31 +60,10 @@ public class MavenMetaAnalyzer extends AbstractMetaAnalyzer {

private static final Logger LOGGER = Logger.getLogger(MavenMetaAnalyzer.class);
private static final String DEFAULT_BASE_URL = "https://repo1.maven.org/maven2";
private static final String REPO_METADATA_URL = "/%s/maven-metadata.xml";
protected static final String UNSTABLE_VERSIONS_PATTERN = "(?i).*[-_\\.](atlassian|preview|snapshot|a|alpha|b|beta|rc|m|ea)[-_\\.]?[0-9]*"; // ignore case

MavenMetaAnalyzer() {
this.baseUrl = DEFAULT_BASE_URL;
}

/**
* Parse two version strings and return the one containing the highest version
*
* @param v1string first version to compare
* @param v2string second version to compare
* @return the highest of two versions as string value
*/
private String highestVersion(String v1string, String v2string) {
if (v1string == null) {
return v2string;
} else if (v2string == null) {
return v1string;
} else {
ComparableVersion v1 = new ComparableVersion(v1string);
ComparableVersion v2 = new ComparableVersion(v2string);
return v1.compareTo(v2) > 0 ? v1string : v2string;
}
}

/**
* {@inheritDoc}
Expand All @@ -107,8 +85,8 @@ public RepositoryType supportedRepositoryType() {
public MetaModel analyze(final Component component) {
final MetaModel meta = new MetaModel(component);
if (component.getPurl() != null) {
final String mavenGavUrl = component.getPurl().getNamespace().replaceAll("\\.", "/") + "/" + component.getPurl().getName();
final String url = String.format(baseUrl + REPO_METADATA_URL, mavenGavUrl);
final String mavenGavUrl = component.getPurl().getNamespace().replace(".", "/") + "/" + component.getPurl().getName();
final String url = String.format("%s%s", baseUrl, String.format("/%s/maven-metadata.xml", mavenGavUrl));
try {
final HttpUriRequest request = new HttpGet(url);

Expand All @@ -122,59 +100,7 @@ public MetaModel analyze(final Component component) {
final HttpEntity entity = response.getEntity();
if (entity != null) {
try (InputStream in = entity.getContent()) {
final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in);
final XPathFactory xpathFactory = XPathFactory.newInstance();
final XPath xpath = xpathFactory.newXPath();

// release: What the latest version in the directory is, of the releases only
final XPathExpression releaseExpression = xpath.compile("/metadata/versioning/release");
final String release = (String) releaseExpression.evaluate(document, XPathConstants.STRING);
final String stableRelease = release.matches(UNSTABLE_VERSIONS_PATTERN) ? null : release;

// latest: What the latest version in the directory is, including snapshots
final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest");
final String latest = (String) latestExpression.evaluate(document, XPathConstants.STRING);
// No need to determine stableLatest as it wil always be null or release:
// final String stableLatest = !latest.matches(UNSTABLE_VERSIONS_PATTERN) ? null : latest;

final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated");
final String lastUpdated = (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING);

if (stableRelease != null) {
meta.setLatestVersion(stableRelease);
if (lastUpdated != null && stableRelease.equals(latest)) {
// lastUpdated reflects the timestamp when latest was updated, so it's only valid when stableRelease == latest
meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated));
}
} else {
// Determine latest stable version from list of versions
// versions/version*: (Many) Versions available of the artifact (both releases and snapshots)
final XPathExpression versionsExpression = xpath.compile("/metadata/versioning/versions/*");
final NodeList versionsList = (NodeList) versionsExpression.evaluate(document, XPathConstants.NODESET);

// collect stable versions
List<String> stableVersions = new ArrayList<>();
for (int n = 0; n < versionsList.getLength(); n++) {
Node versionNode = versionsList.item(n);
String version = versionNode.getFirstChild().getNodeValue().trim();
if (!version.matches(UNSTABLE_VERSIONS_PATTERN)) {
stableVersions.add(version);
}
}

// find latest stable version
String latestStableVersion = null;
if (stableVersions.size() > 0) {
latestStableVersion = stableVersions.stream().reduce(null, (highest, current) -> highestVersion(highest, current));
}
// use latestStableVersion, or else latest unstable release (e.g. alpha, milestone) or else latest snapshot
String latestVersion = latestStableVersion != null ? latestStableVersion: (release != null ? release: latest);
meta.setLatestVersion(latestVersion);
if (lastUpdated != null && latestVersion.equals(latest)) {
// lastUpdated reflects the timestamp when latest was updated, so it's only valid when latestVersion == latest
meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated));
}
}
analyzeContent(meta, in);
}
}
} else {
Expand All @@ -187,4 +113,76 @@ public MetaModel analyze(final Component component) {
}
return meta;
}

/**
* The maven-metadata.xml files are not updated by Nexus during deployment, they are updated by Maven. It downloads the file, updates it, and then redeploys it.
*
* Maven will update the "release" field only during the following scenarios:
*
* 1. Maven 2.x deploys using -DupdateReleaseInfo=true
* 2. Maven 3.x deploys a non-snapshot artifact
*
* The "latest" field is only intended for plugin resolution, and is only set upon deployment of a maven-plugin artifact, both for Maven 2.x and 3.x regardless whether a release or snapshot gets deployed.
*
* Also, Maven will update these fields with whatever version it is currently deploying, so "latest" and "release" will not necessarily correspond to the highest version number.
*
* https://support.sonatype.com/hc/en-us/articles/213464638-Why-are-the-latest-and-release-tags-in-maven-metadata-xml-not-being-updated-after-deploying-artifacts-
*/
private void analyzeContent(final MetaModel meta, InputStream in)
throws SAXException, IOException, ParserConfigurationException, XPathExpressionException {
final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in);
final XPathFactory xpathFactory = XPathFactory.newInstance();
final XPath xpath = xpathFactory.newXPath();

// latest: What the latest version in the directory is, including snapshots
final String latest = getLatestVersionFromMetadata(document, xpath);
// When the metadata was last updated
final String lastUpdated = getLastUpdatedFromMetadata(document, xpath);
// versions/version*: (Many) Versions available of the artifact (both releases and snapshots)
final NodeList versionsList = getVersionsFromMetadata(document, xpath);

// latest and release might not be the highest version in case of a hotfix on an older release!

// find highest stable or unstable version from list of versions
List<String> versions = getVersions(versionsList);
// find highest stable version from list of versions
String highestStableOrUnstableVersion = AbstractMetaAnalyzer.getHighestVersion(versions);

// find highest stable version
String highestStableVersion = getHighestStableVersion(versions);

// use highestStableVersion, or else latest unstable release (e.g. alpha, milestone) or else latest snapshot
String wantedVersion = highestStableVersion != null ? highestStableVersion: highestStableOrUnstableVersion;
meta.setLatestVersion(wantedVersion);
if (lastUpdated != null && wantedVersion != null && wantedVersion.equals(latest)) {
// lastUpdated reflects the timestamp when latest was updated, so it's only valid when wantedVersion == latest
meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated));
}
}

private NodeList getVersionsFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException {
final XPathExpression versionsExpression = xpath.compile("/metadata/versioning/versions/*");
return (NodeList) versionsExpression.evaluate(document, XPathConstants.NODESET);
}

private String getLastUpdatedFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException {
final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated");
return (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING);
}

private String getLatestVersionFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException {
final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest");
return (String) latestExpression.evaluate(document, XPathConstants.STRING);
}

private List<String> getVersions(final NodeList versionsList) {
List<String> versions = new ArrayList<>();
for (int n = 0; n < versionsList.getLength(); n++) {
Node versionNode = versionsList.item(n);
String version = versionNode.getFirstChild().getNodeValue().trim();
versions.add(version);
}
return versions;
}

}
Loading

0 comments on commit cbfe8c1

Please sign in to comment.