diff --git a/docs/_docs/analysis-types/outdated-components.md b/docs/_docs/analysis-types/outdated-components.md index 8ede02b49b..1077126239 100644 --- a/docs/_docs/analysis-types/outdated-components.md +++ b/docs/_docs/analysis-types/outdated-components.md @@ -27,3 +27,9 @@ from various repositories. Dependency-Track relies on Package URL (PURL) to iden to, the metadata about the component, and uses that data to query the various repositories capable of supporting the components ecosystem. Refer to [Repositories]({{ site.baseurl }}{% link _docs/datasources/repositories.md %}) for further information. + +### Stable releases +In some repositories, for example NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a stable release or an unstable version. In NPM as well as Maven repositories, the latest version does not need to be the highest version. It's just the latest published to the repository. + +For some repositories, Dependency-Track tries to find the highest stable release instead of just the latest version. Refer to [Repositories]({{ site.baseurl }}{% link _docs/datasources/repositories.md %}) for further information. + diff --git a/docs/_docs/datasources/repositories.md b/docs/_docs/datasources/repositories.md index 5ecc964c3d..fe1a887b34 100644 --- a/docs/_docs/datasources/repositories.md +++ b/docs/_docs/datasources/repositories.md @@ -59,3 +59,17 @@ leveraging repositories to identify outdated components. Refer to [Datasource Routing]({{ site.baseurl }}{% link _docs/datasources/routing.md %}) for information on Package URL and the various ways it is used throughout Dependency-Track. + +#### Highest stable release +Dependency-Track identifies outdated components by looking for newer versions of the component. Preferably this should be a higher version, but usualy repositories report the newest version of a component which might not be the highest version. Also some repositories report unstable versions as the latest version. + +Dependency-Track tries find the highest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the highest unstable version is reported. This feature is suported for all default repositories: + * Cargo + * Composer + * Gem + * Go + * Hex + * Maven + * NPM + * NuGet + * PyPi diff --git a/pom.xml b/pom.xml index 374db4bcfd..3980f97cc6 100644 --- a/pom.xml +++ b/pom.xml @@ -227,6 +227,12 @@ lucene-sandbox ${lib.lucene.version} + + + com.vdurmont + semver4j + 3.1.0 + io.pebbletemplates diff --git a/src/main/java/org/dependencytrack/policy/VersionPolicyEvaluator.java b/src/main/java/org/dependencytrack/policy/VersionPolicyEvaluator.java index 904063dff8..1ae81cb833 100644 --- a/src/main/java/org/dependencytrack/policy/VersionPolicyEvaluator.java +++ b/src/main/java/org/dependencytrack/policy/VersionPolicyEvaluator.java @@ -18,14 +18,13 @@ */ package org.dependencytrack.policy; -import alpine.common.logging.Logger; +import java.util.ArrayList; +import java.util.List; import org.dependencytrack.model.Component; import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; import org.dependencytrack.util.ComponentVersion; - -import java.util.ArrayList; -import java.util.List; +import alpine.common.logging.Logger; /** * Evaluates a components version against a policy. @@ -56,10 +55,6 @@ public List evaluate(final Policy policy, final Compon LOGGER.debug("Evaluating component (" + component.getUuid() + ") against policy condition (" + condition.getUuid() + ")"); final var conditionVersion = new ComponentVersion(condition.getValue()); - if (conditionVersion.getVersionParts().isEmpty()) { - LOGGER.warn("Unable to parse version (" + condition.getValue() + " provided by condition"); - continue; - } if (matches(componentVersion, conditionVersion, condition.getOperator())) { violations.add(new PolicyConditionViolation(condition, component)); diff --git a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java index 5e0cee6ec2..5150749f76 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java @@ -18,23 +18,23 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; +import java.io.IOException; +import java.net.URISyntaxException; import org.apache.commons.lang3.StringUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.utils.URIBuilder; +import org.apache.maven.artifact.versioning.ComparableVersion; import org.dependencytrack.common.HttpClientPool; import org.dependencytrack.model.Component; import org.dependencytrack.notification.NotificationConstants; import org.dependencytrack.notification.NotificationGroup; import org.dependencytrack.notification.NotificationScope; import org.dependencytrack.util.HttpUtil; - -import java.io.IOException; -import java.net.URISyntaxException; +import alpine.common.logging.Logger; +import alpine.notification.Notification; +import alpine.notification.NotificationLevel; /** * Base abstract class that all IMetaAnalyzer implementations should likely extend. @@ -49,6 +49,7 @@ public abstract class AbstractMetaAnalyzer implements IMetaAnalyzer { protected String username; protected String password; + /** * {@inheritDoc} */ @@ -108,4 +109,29 @@ protected CloseableHttpResponse processHttpRequest(String url) throws IOExceptio } } + /** + * Parse two version strings and return the one containing the highest version + * + * @param v1string first version to compare + * @param v2string second version to compare + * @return the highest of two versions as string value + */ + public static String highestVersion(String v1string, String v2string) { + if (v1string == null) { + return v2string; + } else if (v2string == null) { + return v1string; + } else { + ComparableVersion v1 = new ComparableVersion(stripLeadingV(v1string)); + ComparableVersion v2 = new ComparableVersion(stripLeadingV(v2string)); + return v1.compareTo(v2) > 0 ? v1string : v2string; + } + } + + protected static String stripLeadingV(String s) { + return s.startsWith("v") + ? s.substring(1) + : s; + } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/CargoMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/CargoMetaAnalyzer.java index 2e30a5f650..c6056c3a3f 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/CargoMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/CargoMetaAnalyzer.java @@ -18,20 +18,19 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; -import org.dependencytrack.exception.MetaAnalyzerException; -import org.json.JSONArray; -import org.json.JSONObject; +import java.io.IOException; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; +import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; import org.dependencytrack.util.DateUtil; - -import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; +import com.github.packageurl.PackageURL; +import alpine.common.logging.Logger; /** * An IMetaAnalyzer implementation that supports Cargo via crates.io compatible repos @@ -74,28 +73,11 @@ public MetaModel analyze(final Component component) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { final HttpEntity entity = response.getEntity(); if (entity != null) { - String responseString = EntityUtils.toString(entity); - var jsonObject = new JSONObject(responseString); + final String responseString = EntityUtils.toString(entity); + final JSONObject jsonObject = new JSONObject(responseString); final JSONObject crate = jsonObject.optJSONObject("crate"); - if (crate != null) { - final String latest = crate.getString("newest_version"); - meta.setLatestVersion(latest); - } final JSONArray versions = jsonObject.optJSONArray("versions"); - if (versions != null) { - for (int i = 0; i < versions.length(); i++) { - final JSONObject version = versions.getJSONObject(i); - final String versionString = version.optString("num"); - if (meta.getLatestVersion() != null && meta.getLatestVersion().equals(versionString)) { - final String publishedTimestamp = version.optString("created_at"); - try { - meta.setPublishedTimestamp(DateUtil.fromISO8601(publishedTimestamp)); - } catch (IllegalArgumentException e) { - LOGGER.warn("An error occurred while parsing published time", e); - } - } - } - } + analyzeCrate(meta, crate, versions); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component); @@ -108,4 +90,26 @@ public MetaModel analyze(final Component component) { } return meta; } + + private void analyzeCrate(final MetaModel meta, final JSONObject crate, final JSONArray versions) { + if (crate != null) { + // Cargo has a highest stable version: https://github.com/rust-lang/crates.io/pull/3163 + final String latest = crate.getString("max_stable_version"); + meta.setLatestVersion(latest); + } + if (versions != null) { + for (int i=0; i { - String key = (String) key_; - if (key.startsWith("dev-") || key.endsWith("-dev")) { + // get list of versions and published timestamps + Map versions = new HashMap<>(); + composerPackage.names().forEach(keyObject -> { + String key = (String) keyObject; + if (!key.startsWith("dev-") && !key.endsWith("-dev")) { // dev versions are excluded, since they are not pinned but a VCS-branch. - return; - } - - final String version_normalized = composerPackage.getJSONObject(key).getString("version_normalized"); - ComparableVersion currentComparableVersion = new ComparableVersion(version_normalized); - if (currentComparableVersion.compareTo(latestVersion) < 0) { - // smaller version can be skipped - return; + final String version = composerPackage.getJSONObject(key).getString("version"); + final String published = composerPackage.getJSONObject(key).getString("time"); + versions.put(version, published); } + }); + final String highestVersion = ComponentVersion.findHighestVersion(new ArrayList<>(versions.keySet())); + meta.setLatestVersion(highestVersion); - final String version = composerPackage.getJSONObject(key).getString("version"); - latestVersion.parseVersion(stripLeadingV(version_normalized)); - meta.setLatestVersion(version); + final String published = versions.get(highestVersion); + meta.setPublishedTimestamp(getPublisedTimestamp(published)); - final String published = composerPackage.getJSONObject(key).getString("time"); - try { - meta.setPublishedTimestamp(dateFormat.parse(published)); - } catch (ParseException e) { - LOGGER.warn("An error occurred while parsing upload time", e); - } - }); } catch (IOException ex) { handleRequestException(LOGGER, ex); } catch (Exception ex) { @@ -140,9 +132,15 @@ public MetaModel analyze(final Component component) { return meta; } - private static String stripLeadingV(String s) { - return s.startsWith("v") - ? s.substring(1) - : s; + private Date getPublisedTimestamp(final String published) { + Date publishedTimestamp = null; + try { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); + publishedTimestamp = dateFormat.parse(published); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing upload time", e); + } + return publishedTimestamp; } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/GemMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/GemMetaAnalyzer.java index 375d06a340..a4f37131cf 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/GemMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/GemMetaAnalyzer.java @@ -18,17 +18,25 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; +import org.dependencytrack.util.ComponentVersion; +import org.json.JSONArray; import org.json.JSONObject; - -import java.io.IOException; +import com.github.packageurl.PackageURL; +import alpine.common.logging.Logger; /** * An IMetaAnalyzer implementation that supports Ruby Gems. @@ -40,7 +48,7 @@ public class GemMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(GemMetaAnalyzer.class); private static final String DEFAULT_BASE_URL = "https://rubygems.org"; - private static final String API_URL = "/api/v1/versions/%s/latest.json"; + private static final String API_URL = "/api/v1/versions/%s.json"; GemMetaAnalyzer() { this.baseUrl = DEFAULT_BASE_URL; @@ -71,9 +79,8 @@ public MetaModel analyze(final Component component) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK){ if(response.getEntity()!=null){ String responseString = EntityUtils.toString(response.getEntity()); - var jsonObject = new JSONObject(responseString); - final String latest = jsonObject.getString("version"); - meta.setLatestVersion(latest); + var releasesArray = new JSONArray(responseString); + analyzeReleases(meta, releasesArray); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component); @@ -87,4 +94,30 @@ public MetaModel analyze(final Component component) { } return meta; } + + private void analyzeReleases(final MetaModel meta, final JSONArray releasesArray) { + Map versions = new HashMap<>(); + for (int i = 0; i(versions.keySet())); + meta.setLatestVersion(highestVersion); + + final String createdAt = versions.get(highestVersion); + meta.setPublishedTimestamp(getPublishedTimestamp(createdAt)); + } + + private Date getPublishedTimestamp(final String insertedAt) { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + Date publishedTimestamp = null; + try { + publishedTimestamp = dateFormat.parse(insertedAt); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing published time", e); + } + return publishedTimestamp; + } } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/GoModulesMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/GoModulesMetaAnalyzer.java index e237e6cb40..d2481a25c6 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/GoModulesMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/GoModulesMetaAnalyzer.java @@ -18,8 +18,10 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; + import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -29,9 +31,9 @@ import org.dependencytrack.model.RepositoryType; import org.json.JSONObject; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import com.github.packageurl.PackageURL; + +import alpine.common.logging.Logger; /** * @see GOPROXY protocol @@ -41,7 +43,7 @@ public class GoModulesMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(GoModulesMetaAnalyzer.class); private static final String DEFAULT_BASE_URL = "https://proxy.golang.org"; - private static final String API_URL = "/%s/%s/@latest"; + private static final String API_URL = "/%s/%s/@latest"; // latest selects the highest available release version GoModulesMetaAnalyzer() { this.baseUrl = DEFAULT_BASE_URL; @@ -69,24 +71,9 @@ public MetaModel analyze(final Component component) { try (final CloseableHttpResponse response = processHttpRequest(url)) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { if (response.getEntity()!=null) { - String responseString = EntityUtils.toString(response.getEntity()); - final var responseJson = new JSONObject(responseString); - meta.setLatestVersion(responseJson.getString("Version")); - - // Module versions are prefixed with "v" in the Go ecosystem. - // Because some services (like OSS Index as of July 2021) do not support - // versions with this prefix, components in DT may not be prefixed either. - // - // In order to make the versions comparable still, we strip the "v" prefix as well, - // if it was done for the analyzed component. - if (component.getVersion() != null && !component.getVersion().startsWith("v")) { - meta.setLatestVersion(StringUtils.stripStart(meta.getLatestVersion(), "v")); - } - - final String commitTimestamp = responseJson.getString("Time"); - if (StringUtils.isNotBlank(commitTimestamp)) { // Time is optional - meta.setPublishedTimestamp(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(commitTimestamp)); - } + final String responseString = EntityUtils.toString(response.getEntity()); + final JSONObject responseJson = new JSONObject(responseString); + analyzeResponse(component, meta, responseJson); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component); @@ -100,6 +87,26 @@ public MetaModel analyze(final Component component) { return meta; } + private void analyzeResponse(final Component component, final MetaModel meta, final JSONObject responseJson) + throws ParseException { + meta.setLatestVersion(responseJson.getString("Version")); + + // Module versions are prefixed with "v" in the Go ecosystem. + // Because some services (like OSS Index as of July 2021) do not support + // versions with this prefix, components in DT may not be prefixed either. + // + // In order to make the versions comparable still, we strip the "v" prefix as well, + // if it was done for the analyzed component. + if (component.getVersion() != null && !component.getVersion().startsWith("v")) { + meta.setLatestVersion(StringUtils.stripStart(meta.getLatestVersion(), "v")); + } + + final String commitTimestamp = responseJson.getString("Time"); + if (StringUtils.isNotBlank(commitTimestamp)) { // Time is optional + meta.setPublishedTimestamp(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(commitTimestamp)); + } + } + /** * "To avoid ambiguity when serving from case-insensitive file systems, the $module [...] elements are * case-encoded by replacing every uppercase letter with an exclamation mark followed by the corresponding diff --git a/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java index 942e78cc65..1726cb76f2 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java @@ -18,22 +18,25 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; +import org.dependencytrack.util.ComponentVersion; import org.json.JSONArray; import org.json.JSONObject; - -import java.io.IOException; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; +import com.github.packageurl.PackageURL; +import alpine.common.logging.Logger; /** * An IMetaAnalyzer implementation that supports Hex. @@ -72,12 +75,7 @@ public MetaModel analyze(final Component component) { final MetaModel meta = new MetaModel(component); if (component.getPurl() != null) { - final String packageName; - if (component.getPurl().getNamespace() != null) { - packageName = component.getPurl().getNamespace().replace("@", "%40") + "%2F" + component.getPurl().getName(); - } else { - packageName = component.getPurl().getName(); - } + final String packageName = getPackageName(component.getPurl()); final String url = String.format(baseUrl + API_URL, packageName); try (final CloseableHttpResponse response = processHttpRequest(url)) { @@ -85,23 +83,8 @@ public MetaModel analyze(final Component component) { if (response.getEntity()!=null) { String responseString = EntityUtils.toString(response.getEntity()); var jsonObject = new JSONObject(responseString); - final JSONArray releasesArray = jsonObject.getJSONArray("releases"); - if (releasesArray.length() > 0) { - // The first one in the array is always the latest version - final JSONObject release = releasesArray.getJSONObject(0); - final String latest = release.optString("version", null); - meta.setLatestVersion(latest); - final String insertedAt = release.optString("inserted_at", null); - if (insertedAt != null) { - final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - try { - final Date published = dateFormat.parse(insertedAt); - meta.setPublishedTimestamp(published); - } catch (ParseException e) { - LOGGER.warn("An error occurred while parsing published time", e); - } - } - } + var releasesArray = jsonObject.getJSONArray("releases"); + analyzeReleases(meta, releasesArray); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component); @@ -115,4 +98,39 @@ public MetaModel analyze(final Component component) { return meta; } + private String getPackageName(final PackageURL pUrl) { + if (pUrl.getNamespace() != null) { + return pUrl.getNamespace().replace("@", "%40") + "%2F" + pUrl.getName(); + } else { + return pUrl.getName(); + } + } + + private void analyzeReleases(final MetaModel meta, final JSONArray releasesArray) { + // get list of versions and published timestamps + Map versions = new HashMap<>(); + for (int i = 0; i(versions.keySet())); + meta.setLatestVersion(highestVersion); + + final String insertedAt = versions.get(highestVersion); + meta.setPublishedTimestamp(getPublishedTimestamp(insertedAt)); + } + + private Date getPublishedTimestamp(final String insertedAt) { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + Date publishedTimestamp = null; + try { + publishedTimestamp = dateFormat.parse(insertedAt); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing published time", e); + } + return publishedTimestamp; + } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java index 302e1c0fa9..b95d042de6 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java @@ -18,27 +18,31 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; +import org.dependencytrack.util.ComponentVersion; import org.dependencytrack.util.DateUtil; import org.dependencytrack.util.XmlUtil; import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import java.io.IOException; -import java.io.InputStream; +import com.github.packageurl.PackageURL; +import alpine.common.logging.Logger; /** * An IMetaAnalyzer implementation that supports Maven repositories (including Maven Central). @@ -76,29 +80,14 @@ public RepositoryType supportedRepositoryType() { public MetaModel analyze(final Component component) { final MetaModel meta = new MetaModel(component); if (component.getPurl() != null) { - final String mavenGavUrl = component.getPurl().getNamespace().replaceAll("\\.", "/") + "/" + component.getPurl().getName(); + final String mavenGavUrl = component.getPurl().getNamespace().replace(".", "/") + "/" + component.getPurl().getName(); final String url = String.format(baseUrl + REPO_METADATA_URL, mavenGavUrl); try (final CloseableHttpResponse response = processHttpRequest(url)) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { final HttpEntity entity = response.getEntity(); if (entity != null) { try (InputStream in = entity.getContent()) { - final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in); - final var xpathFactory = XPathFactory.newInstance(); - final XPath xpath = xpathFactory.newXPath(); - - final XPathExpression releaseExpression = xpath.compile("/metadata/versioning/release"); - final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest"); - final var release = (String) releaseExpression.evaluate(document, XPathConstants.STRING); - final String latest = (String) latestExpression.evaluate(document, XPathConstants.STRING); - - final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated"); - final var lastUpdated = (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING); - - meta.setLatestVersion(release != null ? release : latest); - if (lastUpdated != null) { - meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); - } + analyzeContent(meta, in); } } } else { @@ -113,4 +102,70 @@ public MetaModel analyze(final Component component) { } return meta; } + + /** + * The maven-metadata.xml files are not updated by Nexus during deployment, they are updated by Maven. It downloads the file, updates it, and then redeploys it. + * + * Maven will update the "release" field only during the following scenarios: + * + * 1. Maven 2.x deploys using -DupdateReleaseInfo=true + * 2. Maven 3.x deploys a non-snapshot artifact + * + * The "latest" field is only intended for plugin resolution, and is only set upon deployment of a maven-plugin artifact, both for Maven 2.x and 3.x regardless whether a release or snapshot gets deployed. + * + * Also, Maven will update these fields with whatever version it is currently deploying, so "latest" and "release" will not necessarily correspond to the highest version number. + * + * https://support.sonatype.com/hc/en-us/articles/213464638-Why-are-the-latest-and-release-tags-in-maven-metadata-xml-not-being-updated-after-deploying-artifacts- + */ + private void analyzeContent(final MetaModel meta, InputStream in) + throws SAXException, IOException, ParserConfigurationException, XPathExpressionException { + + final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in); + final XPathFactory xpathFactory = XPathFactory.newInstance(); + final XPath xpath = xpathFactory.newXPath(); + + // latest: What the latest version in the directory is, including snapshots + final String latest = getLatestVersionFromMetadata(document, xpath); + // When the metadata was last updated + final String lastUpdated = getLastUpdatedFromMetadata(document, xpath); + // versions/version*: (Many) Versions available of the artifact (both releases and snapshots) + final NodeList versionsList = getVersionsFromMetadata(document, xpath); + + // latest and release might not be the highest version in case of a hotfix on an older release! + + // find highest stable or unstable version from list of versions + List versions = getVersions(versionsList); + String highestVersion = ComponentVersion.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); + if (lastUpdated != null && highestVersion != null && highestVersion.equals(latest)) { + // lastUpdated reflects the timestamp when latest was updated, so it's only valid when highestVersion == latest + meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); + } + } + + private NodeList getVersionsFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression versionsExpression = xpath.compile("/metadata/versioning/versions/*"); + return (NodeList) versionsExpression.evaluate(document, XPathConstants.NODESET); + } + + private String getLastUpdatedFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated"); + return (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING); + } + + private String getLatestVersionFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest"); + return (String) latestExpression.evaluate(document, XPathConstants.STRING); + } + + private List getVersions(final NodeList versionsList) { + List versions = new ArrayList<>(); + for (int n = 0; n < versionsList.getLength(); n++) { + Node versionNode = versionsList.item(n); + String version = versionNode.getFirstChild().getNodeValue().trim(); + versions.add(version); + } + return versions; + } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java index 1142712aa3..4c5445d460 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java @@ -18,17 +18,19 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; +import org.dependencytrack.util.ComponentVersion; import org.json.JSONObject; - -import java.io.IOException; +import com.github.packageurl.PackageURL; +import alpine.common.logging.Logger; /** * An IMetaAnalyzer implementation that supports NPM. @@ -40,7 +42,7 @@ public class NpmMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(NpmMetaAnalyzer.class); private static final String DEFAULT_BASE_URL = "https://registry.npmjs.org"; - private static final String API_URL = "/-/package/%s/dist-tags"; + private static final String API_URL = "/%s/"; NpmMetaAnalyzer() { this.baseUrl = DEFAULT_BASE_URL; @@ -78,12 +80,9 @@ public MetaModel analyze(final Component component) { try (final CloseableHttpResponse response = processHttpRequest(url)) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { if (response.getEntity()!=null) { - String responseString = EntityUtils.toString(response.getEntity()); - var jsonObject = new JSONObject(responseString); - final String latest = jsonObject.optString("latest"); - if (latest != null) { - meta.setLatestVersion(latest); - } + final String responseString = EntityUtils.toString(response.getEntity()); + final JSONObject responseObject = new JSONObject(responseString); + analyzeResponse(meta, responseObject); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component); @@ -97,4 +96,11 @@ public MetaModel analyze(final Component component) { return meta; } + private void analyzeResponse(final MetaModel meta, JSONObject response) { + final JSONObject versionsObject = response.getJSONObject("versions"); + final List versions = new ArrayList<>(versionsObject.keySet()); + final String highestVersion = ComponentVersion.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); + } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java index cf4343cab5..a17e6b5e59 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java @@ -18,23 +18,23 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; -import org.apache.maven.artifact.versioning.ComparableVersion; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; +import org.dependencytrack.util.ComponentVersion; import org.json.JSONArray; import org.json.JSONObject; - -import java.io.IOException; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; +import com.github.packageurl.PackageURL; +import alpine.common.logging.Logger; /** * An IMetaAnalyzer implementation that supports Nuget. @@ -44,7 +44,7 @@ */ public class NugetMetaAnalyzer extends AbstractMetaAnalyzer { - public static final DateFormat[] SUPPORTED_DATE_FORMATS = new DateFormat[]{ + private static final DateFormat[] SUPPORTED_DATE_FORMATS = new DateFormat[]{ new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") }; @@ -95,10 +95,8 @@ public RepositoryType supportedRepositoryType() { */ public MetaModel analyze(final Component component) { final MetaModel meta = new MetaModel(component); - if (component.getPurl() != null) { - if (performVersionCheck(meta, component)) { - performLastPublishedCheck(meta, component); - } + if ((component.getPurl() != null) && performVersionCheck(meta, component)) { + performLastPublishedCheck(meta, component); } return meta; } @@ -108,11 +106,12 @@ private boolean performVersionCheck(final MetaModel meta, final Component compon try (final CloseableHttpResponse response = processHttpRequest(url)) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { if (response.getEntity() != null) { - String responseString = EntityUtils.toString(response.getEntity()); - var jsonObject = new JSONObject(responseString); - final JSONArray versions = jsonObject.getJSONArray("versions"); - final String latest = findLatestVersion(versions); // get the last version in the array - meta.setLatestVersion(latest); + final String responseString = EntityUtils.toString(response.getEntity()); + final JSONObject jsonObject = new JSONObject(responseString); + final JSONArray versionsArray = jsonObject.getJSONArray("versions"); + final List versions = versionsArray.toList().stream().map(Object::toString).toList(); + final String highestVersion = ComponentVersion.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); } return true; } else { @@ -126,23 +125,6 @@ private boolean performVersionCheck(final MetaModel meta, final Component compon return false; } - private String findLatestVersion(JSONArray versions) { - if (versions.length() < 1) { - return null; - } - - ComparableVersion latestVersion = new ComparableVersion(versions.getString(0)); - - for (int i = 1; i < versions.length(); i++) { - ComparableVersion version = new ComparableVersion(versions.getString(i)); - if (version.compareTo(latestVersion) > 0) { - latestVersion = version; - } - } - - return latestVersion.toString(); - } - private boolean performLastPublishedCheck(final MetaModel meta, final Component component) { final String url = String.format(registrationUrl, component.getPurl().getName().toLowerCase(), meta.getLatestVersion()); try (final CloseableHttpResponse response = processHttpRequest(url)) { @@ -173,17 +155,15 @@ private void initializeEndpoints() { final String url = baseUrl + INDEX_URL; try { try (final CloseableHttpResponse response = processHttpRequest(url)) { - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - if(response.getEntity()!=null){ - String responseString = EntityUtils.toString(response.getEntity()); - JSONObject responseJson = new JSONObject(responseString); - final JSONArray resources = responseJson.getJSONArray("resources"); - final JSONObject packageBaseResource = findResourceByType(resources, "PackageBaseAddress"); - final JSONObject registrationsBaseResource = findResourceByType(resources, "RegistrationsBaseUrl"); - if (packageBaseResource != null && registrationsBaseResource != null) { - versionQueryUrl = packageBaseResource.getString("@id") + "%s/index.json"; - registrationUrl = registrationsBaseResource.getString("@id") + "%s/%s.json"; - } + if ((response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) && (response.getEntity() != null)) { + final String responseString = EntityUtils.toString(response.getEntity()); + final JSONObject responseJson = new JSONObject(responseString); + final JSONArray resources = responseJson.getJSONArray("resources"); + final JSONObject packageBaseResource = findResourceByType(resources, "PackageBaseAddress"); + final JSONObject registrationsBaseResource = findResourceByType(resources, "RegistrationsBaseUrl"); + if (packageBaseResource != null && registrationsBaseResource != null) { + versionQueryUrl = packageBaseResource.getString("@id") + "%s/index.json"; + registrationUrl = registrationsBaseResource.getString("@id") + "%s/%s.json"; } } } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java index 2c950bc084..1bd398f803 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java @@ -18,22 +18,23 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import com.github.packageurl.PackageURL; +import java.io.IOException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; import org.dependencytrack.exception.MetaAnalyzerException; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; -import org.json.JSONArray; +import org.dependencytrack.util.ComponentVersion; import org.json.JSONObject; - -import java.io.IOException; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; +import com.github.packageurl.PackageURL; +import alpine.common.logging.Logger; /** * An IMetaAnalyzer implementation that supports Pypi. @@ -75,28 +76,10 @@ public MetaModel analyze(final Component component) { try (final CloseableHttpResponse response = processHttpRequest(url)) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { if (response.getEntity() != null) { - String stringResponse = EntityUtils.toString(response.getEntity()); - JSONObject jsonObject = new JSONObject(stringResponse); - final JSONObject info = jsonObject.getJSONObject("info"); - final String latest = info.optString("version", null); - if (latest != null) { - meta.setLatestVersion(latest); - final JSONObject releases = jsonObject.getJSONObject("releases"); - final JSONArray latestArray = releases.getJSONArray(latest); - if (latestArray.length() > 0) { - final JSONObject release = latestArray.getJSONObject(0); - final String updateTime = release.optString("upload_time", null); - if (updateTime != null) { - final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - try { - final Date published = dateFormat.parse(updateTime); - meta.setPublishedTimestamp(published); - } catch (ParseException e) { - LOGGER.warn("An error occurred while parsing upload time", e); - } - } - } - } + final String stringResponse = EntityUtils.toString(response.getEntity()); + final JSONObject jsonObject = new JSONObject(stringResponse); + final JSONObject releases = jsonObject.getJSONObject("releases"); + analyzeReleases(meta, releases); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component); @@ -109,4 +92,29 @@ public MetaModel analyze(final Component component) { } return meta; } -} \ No newline at end of file + + private void analyzeReleases(final MetaModel meta, final JSONObject releases) { + List versions = new ArrayList<>(releases.keySet()); + final String highestVersion = ComponentVersion.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); + if (highestVersion != null) { + final JSONObject release = releases.getJSONArray(highestVersion).getJSONObject(0); + final String updateTime = release.optString("upload_time", null); + if (updateTime != null) { + Date published = getPublishedTimestamp(updateTime); + meta.setPublishedTimestamp(published); + } + } + } + + private Date getPublishedTimestamp(final String updateTime) { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + Date published = null; + try { + published = dateFormat.parse(updateTime); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing upload time", e); + } + return published; + } +} diff --git a/src/main/java/org/dependencytrack/tasks/scanners/AbstractVulnerableSoftwareAnalysisTask.java b/src/main/java/org/dependencytrack/tasks/scanners/AbstractVulnerableSoftwareAnalysisTask.java index 6937f63dba..856b457f7b 100644 --- a/src/main/java/org/dependencytrack/tasks/scanners/AbstractVulnerableSoftwareAnalysisTask.java +++ b/src/main/java/org/dependencytrack/tasks/scanners/AbstractVulnerableSoftwareAnalysisTask.java @@ -27,7 +27,6 @@ import org.dependencytrack.util.NotificationUtil; import us.springett.parsers.cpe.Cpe; import us.springett.parsers.cpe.values.LogicalValue; - import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.List; @@ -72,19 +71,25 @@ protected void analyzeVersionRange(final QueryManager qm, final Listtrue if the target version is matched; otherwise * false * * Ported from Dependency-Check v5.2.1 */ private static boolean compareVersions(VulnerableSoftware vs, String targetVersion) { + // For VulnerableSoftware (could actually be hardware) without a version number. + // e.g. cpe:2.3:o:intel:2000e_firmware:-:*:*:*:*:*:*:* + if (LogicalValue.NA.getAbbreviation().equals(vs.getVersion())) { + return true; + } //if any of the four conditions will be evaluated - then true; boolean result = (vs.getVersionEndExcluding() != null && !vs.getVersionEndExcluding().isEmpty()) || (vs.getVersionStartExcluding() != null && !vs.getVersionStartExcluding().isEmpty()) || (vs.getVersionEndIncluding() != null && !vs.getVersionEndIncluding().isEmpty()) || (vs.getVersionStartIncluding() != null && !vs.getVersionStartIncluding().isEmpty()); + // Check for CPE wilcards first, before comparing versions // Modified from original by Steve Springett // Added null check: vs.getVersion() != null as purl sources that use version ranges may not have version populated. if (!result && vs.getVersion() != null && compareAttributes(vs.getVersion(), targetVersion)) { @@ -92,9 +97,6 @@ private static boolean compareVersions(VulnerableSoftware vs, String targetVersi } final ComponentVersion target = new ComponentVersion(targetVersion); - if (target.getVersionParts().isEmpty()) { - return false; - } if (result && vs.getVersionEndExcluding() != null && !vs.getVersionEndExcluding().isEmpty()) { final ComponentVersion endExcluding = new ComponentVersion(vs.getVersionEndExcluding()); result = endExcluding.compareTo(target) > 0; diff --git a/src/main/java/org/dependencytrack/upgrade/v490/v490Updater.java b/src/main/java/org/dependencytrack/upgrade/v490/v490Updater.java index 5b4db59e09..e14ee26712 100644 --- a/src/main/java/org/dependencytrack/upgrade/v490/v490Updater.java +++ b/src/main/java/org/dependencytrack/upgrade/v490/v490Updater.java @@ -21,10 +21,9 @@ import alpine.common.logging.Logger; import alpine.persistence.AlpineQueryManager; import alpine.server.upgrade.AbstractUpgradeItem; - +import alpine.server.util.DbUtil; import java.sql.Connection; import java.sql.PreparedStatement; - import static org.dependencytrack.model.ConfigPropertyConstants.SCANNER_SNYK_API_VERSION; public class v490Updater extends AbstractUpgradeItem { @@ -39,6 +38,7 @@ public String getSchemaVersion() { @Override public void executeUpgrade(final AlpineQueryManager qm, final Connection connection) throws Exception { updateDefaultSnykApiVersion(connection); + removeUnstableVersionsFromAnalysisCacheAndRepoMetadata(connection); } /** @@ -62,4 +62,20 @@ private static void updateDefaultSnykApiVersion(final Connection connection) thr } } + /** + * Versions with a '-' in it probably indicate unstable versions. Remove them all + * from component analysis cache and repository metadata, so only stable versions + * remain. + * + * @param connection The {@link Connection} to use for executing queries + * @throws Exception When executing a query failed + * @see: https://github.com/DependencyTrack/dependency-track/issues/2500 + */ + private void removeUnstableVersionsFromAnalysisCacheAndRepoMetadata(Connection connection) throws Exception { + LOGGER.info("Removing possible unstable versions from component analysis cache"); + DbUtil.executeUpdate(connection, "DELETE FROM \"COMPONENTANALYSISCACHE\" WHERE RESULT LIKE '%-%'"); + LOGGER.info("Removing possible unstable versions from repository metadata"); + DbUtil.executeUpdate(connection, "DELETE FROM \"REPOSITORY_META_COMPONENT\" WHERE LATEST_VERSION LIKE '%-%'"); + } + } diff --git a/src/main/java/org/dependencytrack/util/ComponentVersion.java b/src/main/java/org/dependencytrack/util/ComponentVersion.java index f31422162f..2c91e1c24f 100644 --- a/src/main/java/org/dependencytrack/util/ComponentVersion.java +++ b/src/main/java/org/dependencytrack/util/ComponentVersion.java @@ -18,115 +18,221 @@ */ package org.dependencytrack.util; -import org.apache.commons.lang3.StringUtils; +import com.vdurmont.semver4j.Semver; import org.apache.commons.lang3.builder.HashCodeBuilder; - -import javax.annotation.concurrent.NotThreadSafe; -import java.util.ArrayList; -import java.util.Iterator; +import org.apache.maven.artifact.versioning.ComparableVersion; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -/** - *

- * Simple object to track the parts of a version number. The parts are contained - * in a List such that version 1.2.3 will be stored as: versionParts[0] = 1; - * versionParts[1] = 2; - * versionParts[2] = 3; - *

- *

- * Note, the parser contained in this class expects the version numbers to be - * separated by periods. If a different separator is used the parser will likely - * fail.

- * - * @author Jeremy Long - * - * Ported from DependencyVersion in Dependency-Check v5.2.1 - */ -@NotThreadSafe -public class ComponentVersion implements Iterable, Comparable { +public class ComponentVersion implements Comparable { + + // Optional epoch part: number before the first : sign + protected static final String EPOCH_PATTERN_STRING = "^(?:(?\\d*):)?"; + + // Optional label: everything after a ~, ^ or - sign, or after a . sign when it doesn't start with a number + protected static final String LABEL_PATTERN_STRING = "(?