Skip to content

Commit

Permalink
Restrict latest versions to stable releases only for Composer, Hex, M…
Browse files Browse the repository at this point in the history 
…aven, NPM, NuGet, PyPi repositories

Signed-off-by: Walter de Boer <[email protected]>
  • Loading branch information
Walter de Boer committed Feb 20, 2023
1 parent 22791bf commit ccb3260
Show file tree
Hide file tree
Showing 16 changed files with 520 additions and 239 deletions.
6 changes: 5 additions & 1 deletion docs/_docs/analysis-types/outdated-components.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,8 @@ components ecosystem. Refer to [Repositories]({{ site.baseurl }}{% link _docs/da
further information.

### Stable releases
In some repositories, such as NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. For Maven repositories Dependency Track tries find the latest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the latest unstable version is reported.
In some repositories, for example NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. In NPM as wel as Maven repositories the latest version does not need to be the highest version. It's just the latest published to the repository.

For some repositories Dependency track tries to find the highest stable release instead of just the latest version. Refer to [Repositories]({{ site.baseurl }}{% link _docs/datasources/repositories.md %}) for
further information.

15 changes: 15 additions & 0 deletions docs/_docs/datasources/repositories.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,18 @@ leveraging repositories to identify outdated components.

Refer to [Datasource Routing]({{ site.baseurl }}{% link _docs/datasources/routing.md %})
for information on Package URL and the various ways it is used throughout Dependency-Track.

#### Highest stable release
Dependency Track identifies outdated components by looking for newer versions of the component. Preferably this should be
a higher version, but usualy repositories report the latest version of a component which is the last updated version. Also
some repositories report unstable versions as the latest version.

Dependency Track tries find the highest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the highest unstable version is reported. This feature is currently suported for the following repositories:
* Composer
* Hex
* Maven
* NPM
* NuGet
* PyPi

For all other repositories the latest version as reported by the repository is used.
92 changes: 89 additions & 3 deletions src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,19 @@
*/
package org.dependencytrack.tasks.repositories;

import alpine.common.logging.Logger;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import org.apache.maven.artifact.versioning.ComparableVersion;
import org.dependencytrack.model.Component;
import org.dependencytrack.notification.NotificationConstants;
import org.dependencytrack.notification.NotificationGroup;
import org.dependencytrack.notification.NotificationScope;

import alpine.common.logging.Logger;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;

/**
* Base abstract class that all IMetaAnalyzer implementations should likely extend.
*
Expand All @@ -35,6 +39,8 @@
*/
public abstract class AbstractMetaAnalyzer implements IMetaAnalyzer {

protected static final String UNSTABLE_VERSIONS_PATTERN = "(?i).*[-_\\.](dev|atlassian|preview|snapshot|a|alpha|b|beta|rc|m|ea)[-_\\.]?\\d*"; // ignore case

protected String baseUrl;

protected String username;
Expand Down Expand Up @@ -83,5 +89,85 @@ protected void handleRequestException(final Logger logger, final Exception e) {
.level(NotificationLevel.ERROR)
);
}

/**
* Parse two version strings and return the one containing the highest version
*
* @param v1string first version to compare
* @param v2string second version to compare
* @return the highest of two versions as string value
*/
protected static String highestVersion(String v1string, String v2string) {
if (v1string == null) {
return v2string;
} else if (v2string == null) {
return v1string;
} else {
ComparableVersion v1 = new ComparableVersion(stripLeadingV(v1string));
ComparableVersion v2 = new ComparableVersion(stripLeadingV(v2string));
return v1.compareTo(v2) > 0 ? v1string : v2string;
}
}

/**
* Determine wether a version string denotes a stable version
*
* @param version the version string
* @return true if the version string denotes a stable version
*/
protected static boolean isStableVersion(String version) {
return !version.matches(UNSTABLE_VERSIONS_PATTERN);
}

/**
* Get the highest version from a list of version strings
*
* @param versions list of version strings
* @return the highest version in the list
*/
protected static String findHighestStableOrUnstableVersion(List<String> versions) {
String highestStableOrUnstableVersion = null;
if (!versions.isEmpty()) {
highestStableOrUnstableVersion = versions.stream().reduce(null, AbstractMetaAnalyzer::highestVersion);
}
return highestStableOrUnstableVersion;
}

/**
* Get the highest stable version from a list of version strings
*
* @param versions list of version strings
* @return the highest version in the list
*/
protected static String findHighestStableVersion(List<String> versions) {
// collect stable versions
List<String> stableVersions = versions.stream().filter(AbstractMetaAnalyzer::isStableVersion).toList();
return findHighestStableOrUnstableVersion(stableVersions);
}


/**
* Get the highest version from a list of version strings. When a stable version is found
* this is returned, otherwise an unstable version or null weh no version is found
*
* @param versions list of version strings
* @return the highest version in the list
*/
protected static String findHighestVersion(List<String> versions) {
// find highest stable version from list of versions
String highestStableOrUnstableVersion = AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(versions);

// find highest stable version
String highestStableVersion = findHighestStableVersion(versions);

// use highestStableVersion, or else latest unstable release (e.g. alpha, milestone) or else latest snapshot
return highestStableVersion != null ? highestStableVersion: highestStableOrUnstableVersion;
}

protected static String stripLeadingV(String s) {
return s.startsWith("v")
? s.substring(1)
: s;
}

}
77 changes: 38 additions & 39 deletions src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,28 @@
*/
package org.dependencytrack.tasks.repositories;

import alpine.common.logging.Logger;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

import org.dependencytrack.common.UnirestFactory;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;

import com.github.packageurl.PackageURL;

import alpine.common.logging.Logger;
import kong.unirest.GetRequest;
import kong.unirest.HttpRequest;
import kong.unirest.HttpResponse;
import kong.unirest.JsonNode;
import kong.unirest.UnirestException;
import kong.unirest.UnirestInstance;
import kong.unirest.json.JSONObject;
import org.apache.maven.artifact.versioning.ComparableVersion;
import org.dependencytrack.common.UnirestFactory;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;

import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;

/**
* An IMetaAnalyzer implementation that supports Composer.
Expand Down Expand Up @@ -80,7 +85,7 @@ public MetaModel analyze(final Component component) {
return meta;
}

final String url = String.format(baseUrl + API_URL, component.getPurl().getNamespace(), component.getPurl().getName());
final String url = String.format(baseUrl.concat(API_URL), component.getPurl().getNamespace(), component.getPurl().getName());
try {
final HttpRequest<GetRequest> request = ui.get(url)
.header("accept", "application/json");
Expand Down Expand Up @@ -109,45 +114,39 @@ public MetaModel analyze(final Component component) {
}
final JSONObject composerPackage = responsePackages.getJSONObject(expectedResponsePackage);

final ComparableVersion latestVersion = new ComparableVersion(stripLeadingV(component.getPurl().getVersion()));
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");

composerPackage.names().forEach(key_ -> {
String key = (String) key_;
if (key.startsWith("dev-") || key.endsWith("-dev")) {
// get list of versions and published timestamps
Map<String, String> versions = new HashMap<>();
composerPackage.names().forEach(keyObject -> {
String key = (String) keyObject;
if (!key.startsWith("dev-") && !key.endsWith("-dev")) {
// dev versions are excluded, since they are not pinned but a VCS-branch.
return;
}

final String version_normalized = composerPackage.getJSONObject(key).getString("version_normalized");
ComparableVersion currentComparableVersion = new ComparableVersion(version_normalized);
if ( currentComparableVersion.compareTo(latestVersion) < 0)
{
// smaller version can be skipped
return;
final String version = composerPackage.getJSONObject(key).getString("version");
final String published = composerPackage.getJSONObject(key).getString("time");
versions.put(version, published);
}
});
final String highestVersion = AbstractMetaAnalyzer.findHighestVersion(new ArrayList<>(versions.keySet()));
meta.setLatestVersion(highestVersion);

final String version = composerPackage.getJSONObject(key).getString("version");
latestVersion.parseVersion(stripLeadingV(version_normalized));
meta.setLatestVersion(version);
final String published = versions.get(highestVersion);
meta.setPublishedTimestamp(getPublisedTimestamp(published));

final String published = composerPackage.getJSONObject(key).getString("time");
try {
meta.setPublishedTimestamp(dateFormat.parse(published));
} catch (ParseException e) {
LOGGER.warn("An error occurred while parsing upload time", e);
}
});
} catch (UnirestException e) {
handleRequestException(LOGGER, e);
}

return meta;
}

private static String stripLeadingV(String s) {
return s.startsWith("v")
? s.substring(1)
: s;
private Date getPublisedTimestamp(final String published) {
Date publishedTimestamp = null;
try {
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
publishedTimestamp = dateFormat.parse(published);
} catch (ParseException e) {
LOGGER.warn("An error occurred while parsing upload time", e);
}
return publishedTimestamp;
}

}
86 changes: 54 additions & 32 deletions src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,21 @@
*/
package org.dependencytrack.tasks.repositories;

import alpine.common.logging.Logger;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

import org.dependencytrack.common.UnirestFactory;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;

import com.github.packageurl.PackageURL;

import alpine.common.logging.Logger;
import kong.unirest.GetRequest;
import kong.unirest.HttpRequest;
import kong.unirest.HttpResponse;
Expand All @@ -28,14 +41,6 @@
import kong.unirest.UnirestInstance;
import kong.unirest.json.JSONArray;
import kong.unirest.json.JSONObject;
import org.dependencytrack.common.UnirestFactory;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;

import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;

/**
* An IMetaAnalyzer implementation that supports Hex.
Expand Down Expand Up @@ -75,14 +80,9 @@ public MetaModel analyze(final Component component) {
final MetaModel meta = new MetaModel(component);
if (component.getPurl() != null) {

final String packageName;
if (component.getPurl().getNamespace() != null) {
packageName = component.getPurl().getNamespace().replace("@", "%40") + "%2F" + component.getPurl().getName();
} else {
packageName = component.getPurl().getName();
}
final String packageName = getPackageName(component);

final String url = String.format(baseUrl + API_URL, packageName);
final String url = String.format(baseUrl.concat(API_URL), packageName);
try {
final HttpRequest<GetRequest> request = ui.get(url)
.header("accept", "application/json");
Expand All @@ -94,22 +94,7 @@ public MetaModel analyze(final Component component) {
if (response.getStatus() == 200) {
if (response.getBody() != null && response.getBody().getObject() != null) {
final JSONArray releasesArray = response.getBody().getObject().getJSONArray("releases");
if (releasesArray.length() > 0) {
// The first one in the array is always the latest version
final JSONObject release = releasesArray.getJSONObject(0);
final String latest = release.optString("version", null);
meta.setLatestVersion(latest);
final String insertedAt = release.optString("inserted_at", null);
if (insertedAt != null) {
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
try {
final Date published = dateFormat.parse(insertedAt);
meta.setPublishedTimestamp(published);
} catch (ParseException e) {
LOGGER.warn("An error occurred while parsing published time", e);
}
}
}
analyzeReleases(meta, releasesArray);
}
} else {
handleUnexpectedHttpResponse(LOGGER, url, response.getStatus(), response.getStatusText(), component);
Expand All @@ -121,4 +106,41 @@ public MetaModel analyze(final Component component) {
return meta;
}

private String getPackageName(final Component component) {
final String packageName;
if (component.getPurl().getNamespace() != null) {
packageName = component.getPurl().getNamespace().replace("@", "%40") + "%2F" + component.getPurl().getName();
} else {
packageName = component.getPurl().getName();
}
return packageName;
}

private void analyzeReleases(final MetaModel meta, final JSONArray releasesArray) {
// get list of versions and published timestamps
Map<String, String> versions = new HashMap<>();
for (int i = 0; i<releasesArray.length(); i++) {
JSONObject release = releasesArray.getJSONObject(i);
final String version = release.optString("version", null);
final String insertedAt = release.optString("inserted_at", null);
versions.put(version, insertedAt);
}
final String highestVersion = AbstractMetaAnalyzer.findHighestVersion(new ArrayList<>(versions.keySet()));
meta.setLatestVersion(highestVersion);

final String insertedAt = versions.get(highestVersion);
meta.setPublishedTimestamp(getPublishedTimestamp(insertedAt));
}

private Date getPublishedTimestamp(final String insertedAt) {
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
Date publishedTimestamp = null;
try {
publishedTimestamp = dateFormat.parse(insertedAt);
} catch (ParseException e) {
LOGGER.warn("An error occurred while parsing published time", e);
}
return publishedTimestamp;
}

}
Loading

0 comments on commit ccb3260

Please sign in to comment.