From ccb3260edce2914387d2551c2dc62420ba36836e Mon Sep 17 00:00:00 2001 From: Walter de Boer Date: Mon, 20 Feb 2023 07:45:20 +0100 Subject: [PATCH] Restrict latest versions to stable releases only for Composer, Hex, Maven, NPM, NuGet, PyPi repositories Signed-off-by: Walter de Boer --- .../analysis-types/outdated-components.md | 6 +- docs/_docs/datasources/repositories.md | 15 ++ .../repositories/AbstractMetaAnalyzer.java | 92 ++++++++++- .../repositories/ComposerMetaAnalyzer.java | 77 +++++---- .../tasks/repositories/HexMetaAnalyzer.java | 86 ++++++---- .../tasks/repositories/MavenMetaAnalyzer.java | 147 +++++++++--------- .../tasks/repositories/NpmMetaAnalyzer.java | 18 ++- .../tasks/repositories/NugetMetaAnalyzer.java | 34 +--- .../tasks/repositories/PypiMetaAnalyzer.java | 72 +++++---- .../AbstractMetaAnalyzerTest.java | 116 ++++++++++++++ .../ComposerMetaAnalyzerTest.java | 2 + .../repositories/HexMetaAnalyzerTest.java | 22 ++- .../repositories/MavenMetaAnalyzerTest.java | 30 ++-- .../repositories/NpmMetaAnalyzerTest.java | 2 +- .../repositories/NugetMetaAnalyzerTest.java | 20 +++ .../repositories/PypiMetaAnalyzerTest.java | 20 ++- 16 files changed, 520 insertions(+), 239 deletions(-) create mode 100644 src/test/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzerTest.java diff --git a/docs/_docs/analysis-types/outdated-components.md b/docs/_docs/analysis-types/outdated-components.md index 1236bfb474..d002d721d4 100644 --- a/docs/_docs/analysis-types/outdated-components.md +++ b/docs/_docs/analysis-types/outdated-components.md @@ -29,4 +29,8 @@ components ecosystem. Refer to [Repositories]({{ site.baseurl }}{% link _docs/da further information. ### Stable releases -In some repositories, such as NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. For Maven repositories Dependency Track tries find the latest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the latest unstable version is reported. +In some repositories, for example NPM, the latest release should always denote a stable release. In others, such as Maven, the latest version might be a a stable release or an unstable version. In NPM as wel as Maven repositories the latest version does not need to be the highest version. It's just the latest published to the repository. + +For some repositories Dependency track tries to find the highest stable release instead of just the latest version. Refer to [Repositories]({{ site.baseurl }}{% link _docs/datasources/repositories.md %}) for +further information. + diff --git a/docs/_docs/datasources/repositories.md b/docs/_docs/datasources/repositories.md index b8644bc5b4..9b73f3c65f 100644 --- a/docs/_docs/datasources/repositories.md +++ b/docs/_docs/datasources/repositories.md @@ -58,3 +58,18 @@ leveraging repositories to identify outdated components. Refer to [Datasource Routing]({{ site.baseurl }}{% link _docs/datasources/routing.md %}) for information on Package URL and the various ways it is used throughout Dependency-Track. + +#### Highest stable release +Dependency Track identifies outdated components by looking for newer versions of the component. Preferably this should be +a higher version, but usualy repositories report the latest version of a component which is the last updated version. Also +some repositories report unstable versions as the latest version. + +Dependency Track tries find the highest stable release by parsing the list of versions and ignoring labels like alpha, beta, or snapshot. When no stable release exists, the highest unstable version is reported. This feature is currently suported for the following repositories: + * Composer + * Hex + * Maven + * NPM + * NuGet + * PyPi + +For all other repositories the latest version as reported by the repository is used. \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java index bca39977dd..6a2d4cdf85 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java @@ -18,15 +18,19 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; -import alpine.notification.Notification; -import alpine.notification.NotificationLevel; +import java.util.List; + import org.apache.commons.lang3.StringUtils; +import org.apache.maven.artifact.versioning.ComparableVersion; import org.dependencytrack.model.Component; import org.dependencytrack.notification.NotificationConstants; import org.dependencytrack.notification.NotificationGroup; import org.dependencytrack.notification.NotificationScope; +import alpine.common.logging.Logger; +import alpine.notification.Notification; +import alpine.notification.NotificationLevel; + /** * Base abstract class that all IMetaAnalyzer implementations should likely extend. * @@ -35,6 +39,8 @@ */ public abstract class AbstractMetaAnalyzer implements IMetaAnalyzer { + protected static final String UNSTABLE_VERSIONS_PATTERN = "(?i).*[-_\\.](dev|atlassian|preview|snapshot|a|alpha|b|beta|rc|m|ea)[-_\\.]?\\d*"; // ignore case + protected String baseUrl; protected String username; @@ -83,5 +89,85 @@ protected void handleRequestException(final Logger logger, final Exception e) { .level(NotificationLevel.ERROR) ); } + + /** + * Parse two version strings and return the one containing the highest version + * + * @param v1string first version to compare + * @param v2string second version to compare + * @return the highest of two versions as string value + */ + protected static String highestVersion(String v1string, String v2string) { + if (v1string == null) { + return v2string; + } else if (v2string == null) { + return v1string; + } else { + ComparableVersion v1 = new ComparableVersion(stripLeadingV(v1string)); + ComparableVersion v2 = new ComparableVersion(stripLeadingV(v2string)); + return v1.compareTo(v2) > 0 ? v1string : v2string; + } + } + + /** + * Determine wether a version string denotes a stable version + * + * @param version the version string + * @return true if the version string denotes a stable version + */ + protected static boolean isStableVersion(String version) { + return !version.matches(UNSTABLE_VERSIONS_PATTERN); + } + + /** + * Get the highest version from a list of version strings + * + * @param versions list of version strings + * @return the highest version in the list + */ + protected static String findHighestStableOrUnstableVersion(List versions) { + String highestStableOrUnstableVersion = null; + if (!versions.isEmpty()) { + highestStableOrUnstableVersion = versions.stream().reduce(null, AbstractMetaAnalyzer::highestVersion); + } + return highestStableOrUnstableVersion; + } + + /** + * Get the highest stable version from a list of version strings + * + * @param versions list of version strings + * @return the highest version in the list + */ + protected static String findHighestStableVersion(List versions) { + // collect stable versions + List stableVersions = versions.stream().filter(AbstractMetaAnalyzer::isStableVersion).toList(); + return findHighestStableOrUnstableVersion(stableVersions); + } + + + /** + * Get the highest version from a list of version strings. When a stable version is found + * this is returned, otherwise an unstable version or null weh no version is found + * + * @param versions list of version strings + * @return the highest version in the list + */ + protected static String findHighestVersion(List versions) { + // find highest stable version from list of versions + String highestStableOrUnstableVersion = AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(versions); + + // find highest stable version + String highestStableVersion = findHighestStableVersion(versions); + + // use highestStableVersion, or else latest unstable release (e.g. alpha, milestone) or else latest snapshot + return highestStableVersion != null ? highestStableVersion: highestStableOrUnstableVersion; + } + + protected static String stripLeadingV(String s) { + return s.startsWith("v") + ? s.substring(1) + : s; + } } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java index 032b00b51c..95ee7668c1 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java @@ -18,8 +18,21 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.dependencytrack.common.UnirestFactory; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; + import com.github.packageurl.PackageURL; + +import alpine.common.logging.Logger; import kong.unirest.GetRequest; import kong.unirest.HttpRequest; import kong.unirest.HttpResponse; @@ -27,14 +40,6 @@ import kong.unirest.UnirestException; import kong.unirest.UnirestInstance; import kong.unirest.json.JSONObject; -import org.apache.maven.artifact.versioning.ComparableVersion; -import org.dependencytrack.common.UnirestFactory; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.RepositoryType; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; /** * An IMetaAnalyzer implementation that supports Composer. @@ -80,7 +85,7 @@ public MetaModel analyze(final Component component) { return meta; } - final String url = String.format(baseUrl + API_URL, component.getPurl().getNamespace(), component.getPurl().getName()); + final String url = String.format(baseUrl.concat(API_URL), component.getPurl().getNamespace(), component.getPurl().getName()); try { final HttpRequest request = ui.get(url) .header("accept", "application/json"); @@ -109,35 +114,23 @@ public MetaModel analyze(final Component component) { } final JSONObject composerPackage = responsePackages.getJSONObject(expectedResponsePackage); - final ComparableVersion latestVersion = new ComparableVersion(stripLeadingV(component.getPurl().getVersion())); - final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); - - composerPackage.names().forEach(key_ -> { - String key = (String) key_; - if (key.startsWith("dev-") || key.endsWith("-dev")) { + // get list of versions and published timestamps + Map versions = new HashMap<>(); + composerPackage.names().forEach(keyObject -> { + String key = (String) keyObject; + if (!key.startsWith("dev-") && !key.endsWith("-dev")) { // dev versions are excluded, since they are not pinned but a VCS-branch. - return; - } - - final String version_normalized = composerPackage.getJSONObject(key).getString("version_normalized"); - ComparableVersion currentComparableVersion = new ComparableVersion(version_normalized); - if ( currentComparableVersion.compareTo(latestVersion) < 0) - { - // smaller version can be skipped - return; + final String version = composerPackage.getJSONObject(key).getString("version"); + final String published = composerPackage.getJSONObject(key).getString("time"); + versions.put(version, published); } + }); + final String highestVersion = AbstractMetaAnalyzer.findHighestVersion(new ArrayList<>(versions.keySet())); + meta.setLatestVersion(highestVersion); - final String version = composerPackage.getJSONObject(key).getString("version"); - latestVersion.parseVersion(stripLeadingV(version_normalized)); - meta.setLatestVersion(version); + final String published = versions.get(highestVersion); + meta.setPublishedTimestamp(getPublisedTimestamp(published)); - final String published = composerPackage.getJSONObject(key).getString("time"); - try { - meta.setPublishedTimestamp(dateFormat.parse(published)); - } catch (ParseException e) { - LOGGER.warn("An error occurred while parsing upload time", e); - } - }); } catch (UnirestException e) { handleRequestException(LOGGER, e); } @@ -145,9 +138,15 @@ public MetaModel analyze(final Component component) { return meta; } - private static String stripLeadingV(String s) { - return s.startsWith("v") - ? s.substring(1) - : s; + private Date getPublisedTimestamp(final String published) { + Date publishedTimestamp = null; + try { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); + publishedTimestamp = dateFormat.parse(published); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing upload time", e); + } + return publishedTimestamp; } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java index 0d5b23dd6b..f1e410cb0f 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzer.java @@ -18,8 +18,21 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.dependencytrack.common.UnirestFactory; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; + import com.github.packageurl.PackageURL; + +import alpine.common.logging.Logger; import kong.unirest.GetRequest; import kong.unirest.HttpRequest; import kong.unirest.HttpResponse; @@ -28,14 +41,6 @@ import kong.unirest.UnirestInstance; import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; -import org.dependencytrack.common.UnirestFactory; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.RepositoryType; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; /** * An IMetaAnalyzer implementation that supports Hex. @@ -75,14 +80,9 @@ public MetaModel analyze(final Component component) { final MetaModel meta = new MetaModel(component); if (component.getPurl() != null) { - final String packageName; - if (component.getPurl().getNamespace() != null) { - packageName = component.getPurl().getNamespace().replace("@", "%40") + "%2F" + component.getPurl().getName(); - } else { - packageName = component.getPurl().getName(); - } + final String packageName = getPackageName(component); - final String url = String.format(baseUrl + API_URL, packageName); + final String url = String.format(baseUrl.concat(API_URL), packageName); try { final HttpRequest request = ui.get(url) .header("accept", "application/json"); @@ -94,22 +94,7 @@ public MetaModel analyze(final Component component) { if (response.getStatus() == 200) { if (response.getBody() != null && response.getBody().getObject() != null) { final JSONArray releasesArray = response.getBody().getObject().getJSONArray("releases"); - if (releasesArray.length() > 0) { - // The first one in the array is always the latest version - final JSONObject release = releasesArray.getJSONObject(0); - final String latest = release.optString("version", null); - meta.setLatestVersion(latest); - final String insertedAt = release.optString("inserted_at", null); - if (insertedAt != null) { - final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - try { - final Date published = dateFormat.parse(insertedAt); - meta.setPublishedTimestamp(published); - } catch (ParseException e) { - LOGGER.warn("An error occurred while parsing published time", e); - } - } - } + analyzeReleases(meta, releasesArray); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatus(), response.getStatusText(), component); @@ -121,4 +106,41 @@ public MetaModel analyze(final Component component) { return meta; } + private String getPackageName(final Component component) { + final String packageName; + if (component.getPurl().getNamespace() != null) { + packageName = component.getPurl().getNamespace().replace("@", "%40") + "%2F" + component.getPurl().getName(); + } else { + packageName = component.getPurl().getName(); + } + return packageName; + } + + private void analyzeReleases(final MetaModel meta, final JSONArray releasesArray) { + // get list of versions and published timestamps + Map versions = new HashMap<>(); + for (int i = 0; i(versions.keySet())); + meta.setLatestVersion(highestVersion); + + final String insertedAt = versions.get(highestVersion); + meta.setPublishedTimestamp(getPublishedTimestamp(insertedAt)); + } + + private Date getPublishedTimestamp(final String insertedAt) { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + Date publishedTimestamp = null; + try { + publishedTimestamp = dateFormat.parse(insertedAt); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing published time", e); + } + return publishedTimestamp; + } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java index 304671a819..9daefe6a16 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzer.java @@ -35,7 +35,6 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; -import org.apache.maven.artifact.versioning.ComparableVersion; import org.dependencytrack.common.HttpClientPool; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; @@ -62,30 +61,10 @@ public class MavenMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(MavenMetaAnalyzer.class); private static final String DEFAULT_BASE_URL = "https://repo1.maven.org/maven2"; private static final String REPO_METADATA_URL = "/%s/maven-metadata.xml"; - protected static final String UNSTABLE_VERSIONS_PATTERN = "(?i).*[-_\\.](atlassian|preview|snapshot|a|alpha|b|beta|rc|m|ea)[-_\\.]?[0-9]*"; // ignore case - + MavenMetaAnalyzer() { this.baseUrl = DEFAULT_BASE_URL; } - - /** - * Parse two version strings and return the one containing the highest version - * - * @param v1string first version to compare - * @param v2string second version to compare - * @return the highest of two versions as string value - */ - private String highestVersion(String v1string, String v2string) { - if (v1string == null) { - return v2string; - } else if (v2string == null) { - return v1string; - } else { - ComparableVersion v1 = new ComparableVersion(v1string); - ComparableVersion v2 = new ComparableVersion(v2string); - return v1.compareTo(v2) > 0 ? v1string : v2string; - } - } /** * {@inheritDoc} @@ -107,8 +86,8 @@ public RepositoryType supportedRepositoryType() { public MetaModel analyze(final Component component) { final MetaModel meta = new MetaModel(component); if (component.getPurl() != null) { - final String mavenGavUrl = component.getPurl().getNamespace().replaceAll("\\.", "/") + "/" + component.getPurl().getName(); - final String url = String.format(baseUrl + REPO_METADATA_URL, mavenGavUrl); + final String mavenGavUrl = component.getPurl().getNamespace().replace(".", "/") + "/" + component.getPurl().getName(); + final String url = String.format(baseUrl.concat(REPO_METADATA_URL), mavenGavUrl); try { final HttpUriRequest request = new HttpGet(url); @@ -122,59 +101,7 @@ public MetaModel analyze(final Component component) { final HttpEntity entity = response.getEntity(); if (entity != null) { try (InputStream in = entity.getContent()) { - final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in); - final XPathFactory xpathFactory = XPathFactory.newInstance(); - final XPath xpath = xpathFactory.newXPath(); - - // release: What the latest version in the directory is, of the releases only - final XPathExpression releaseExpression = xpath.compile("/metadata/versioning/release"); - final String release = (String) releaseExpression.evaluate(document, XPathConstants.STRING); - final String stableRelease = release.matches(UNSTABLE_VERSIONS_PATTERN) ? null : release; - - // latest: What the latest version in the directory is, including snapshots - final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest"); - final String latest = (String) latestExpression.evaluate(document, XPathConstants.STRING); - // No need to determine stableLatest as it wil always be null or release: - // final String stableLatest = !latest.matches(UNSTABLE_VERSIONS_PATTERN) ? null : latest; - - final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated"); - final String lastUpdated = (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING); - - if (stableRelease != null) { - meta.setLatestVersion(stableRelease); - if (lastUpdated != null && stableRelease.equals(latest)) { - // lastUpdated reflects the timestamp when latest was updated, so it's only valid when stableRelease == latest - meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); - } - } else { - // Determine latest stable version from list of versions - // versions/version*: (Many) Versions available of the artifact (both releases and snapshots) - final XPathExpression versionsExpression = xpath.compile("/metadata/versioning/versions/*"); - final NodeList versionsList = (NodeList) versionsExpression.evaluate(document, XPathConstants.NODESET); - - // collect stable versions - List stableVersions = new ArrayList<>(); - for (int n = 0; n < versionsList.getLength(); n++) { - Node versionNode = versionsList.item(n); - String version = versionNode.getFirstChild().getNodeValue().trim(); - if (!version.matches(UNSTABLE_VERSIONS_PATTERN)) { - stableVersions.add(version); - } - } - - // find latest stable version - String latestStableVersion = null; - if (stableVersions.size() > 0) { - latestStableVersion = stableVersions.stream().reduce(null, (highest, current) -> highestVersion(highest, current)); - } - // use latestStableVersion, or else latest unstable release (e.g. alpha, milestone) or else latest snapshot - String latestVersion = latestStableVersion != null ? latestStableVersion: (release != null ? release: latest); - meta.setLatestVersion(latestVersion); - if (lastUpdated != null && latestVersion.equals(latest)) { - // lastUpdated reflects the timestamp when latest was updated, so it's only valid when latestVersion == latest - meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); - } - } + analyzeContent(meta, in); } } } else { @@ -187,4 +114,70 @@ public MetaModel analyze(final Component component) { } return meta; } + + /** + * The maven-metadata.xml files are not updated by Nexus during deployment, they are updated by Maven. It downloads the file, updates it, and then redeploys it. + * + * Maven will update the "release" field only during the following scenarios: + * + * 1. Maven 2.x deploys using -DupdateReleaseInfo=true + * 2. Maven 3.x deploys a non-snapshot artifact + * + * The "latest" field is only intended for plugin resolution, and is only set upon deployment of a maven-plugin artifact, both for Maven 2.x and 3.x regardless whether a release or snapshot gets deployed. + * + * Also, Maven will update these fields with whatever version it is currently deploying, so "latest" and "release" will not necessarily correspond to the highest version number. + * + * https://support.sonatype.com/hc/en-us/articles/213464638-Why-are-the-latest-and-release-tags-in-maven-metadata-xml-not-being-updated-after-deploying-artifacts- + */ + private void analyzeContent(final MetaModel meta, InputStream in) + throws SAXException, IOException, ParserConfigurationException, XPathExpressionException { + + final Document document = XmlUtil.buildSecureDocumentBuilder().parse(in); + final XPathFactory xpathFactory = XPathFactory.newInstance(); + final XPath xpath = xpathFactory.newXPath(); + + // latest: What the latest version in the directory is, including snapshots + final String latest = getLatestVersionFromMetadata(document, xpath); + // When the metadata was last updated + final String lastUpdated = getLastUpdatedFromMetadata(document, xpath); + // versions/version*: (Many) Versions available of the artifact (both releases and snapshots) + final NodeList versionsList = getVersionsFromMetadata(document, xpath); + + // latest and release might not be the highest version in case of a hotfix on an older release! + + // find highest stable or unstable version from list of versions + List versions = getVersions(versionsList); + String highestVersion = AbstractMetaAnalyzer.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); + if (lastUpdated != null && highestVersion != null && highestVersion.equals(latest)) { + // lastUpdated reflects the timestamp when latest was updated, so it's only valid when highestVersion == latest + meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); + } + } + + private NodeList getVersionsFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression versionsExpression = xpath.compile("/metadata/versioning/versions/*"); + return (NodeList) versionsExpression.evaluate(document, XPathConstants.NODESET); + } + + private String getLastUpdatedFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression lastUpdatedExpression = xpath.compile("/metadata/versioning/lastUpdated"); + return (String) lastUpdatedExpression.evaluate(document, XPathConstants.STRING); + } + + private String getLatestVersionFromMetadata(final Document document, final XPath xpath) throws XPathExpressionException { + final XPathExpression latestExpression = xpath.compile("/metadata/versioning/latest"); + return (String) latestExpression.evaluate(document, XPathConstants.STRING); + } + + private List getVersions(final NodeList versionsList) { + List versions = new ArrayList<>(); + for (int n = 0; n < versionsList.getLength(); n++) { + Node versionNode = versionsList.item(n); + String version = versionNode.getFirstChild().getNodeValue().trim(); + versions.add(version); + } + return versions; + } + } diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java index 03ccf30bfc..c84fd7285a 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzer.java @@ -26,6 +26,9 @@ import kong.unirest.JsonNode; import kong.unirest.UnirestException; import kong.unirest.UnirestInstance; +import kong.unirest.json.JSONObject; +import java.util.ArrayList; +import java.util.List; import org.dependencytrack.common.UnirestFactory; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; @@ -40,7 +43,7 @@ public class NpmMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = Logger.getLogger(NpmMetaAnalyzer.class); private static final String DEFAULT_BASE_URL = "https://registry.npmjs.org"; - private static final String API_URL = "/-/package/%s/dist-tags"; + private static final String API_URL = "/%s/"; NpmMetaAnalyzer() { this.baseUrl = DEFAULT_BASE_URL; @@ -75,21 +78,24 @@ public MetaModel analyze(final Component component) { packageName = component.getPurl().getName(); } - final String url = String.format(baseUrl + API_URL, packageName); + final String url = String.format(baseUrl.concat(API_URL), packageName); try { final HttpRequest request = ui.get(url) .header("accept", "application/json"); if (username != null || password != null) { request.basicAuth(username, password); } + final HttpResponse response = request.asJson(); if (response.getStatus() == 200) { if (response.getBody() != null && response.getBody().getObject() != null) { - final String latest = response.getBody().getObject().optString("latest"); - if (latest != null) { - meta.setLatestVersion(latest); - } + + final JSONObject versionsObject = response.getBody().getObject().getJSONObject("versions"); + final List versions = new ArrayList<>(versionsObject.keySet()); + final String highestVersion = AbstractMetaAnalyzer.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); + } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatus(), response.getStatusText(), component); diff --git a/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java index bd0333dc59..4b9d58ceac 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzer.java @@ -28,7 +28,6 @@ import kong.unirest.UnirestInstance; import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; -import org.apache.maven.artifact.versioning.ComparableVersion; import org.dependencytrack.common.UnirestFactory; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; @@ -37,6 +36,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.List; /** * An IMetaAnalyzer implementation that supports Nuget. @@ -46,7 +46,7 @@ */ public class NugetMetaAnalyzer extends AbstractMetaAnalyzer { - public static final DateFormat[] SUPPORTED_DATE_FORMATS = new DateFormat[]{ + private static final DateFormat[] SUPPORTED_DATE_FORMATS = new DateFormat[]{ new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") }; @@ -97,10 +97,8 @@ public RepositoryType supportedRepositoryType() { */ public MetaModel analyze(final Component component) { final MetaModel meta = new MetaModel(component); - if (component.getPurl() != null) { - if (performVersionCheck(meta, component)) { - performLastPublishedCheck(meta, component); - } + if ((component.getPurl() != null) && performVersionCheck(meta, component)) { + performLastPublishedCheck(meta, component); } return meta; } @@ -111,9 +109,10 @@ private boolean performVersionCheck(final MetaModel meta, final Component compon final HttpResponse response = httpGet(url); if (response.getStatus() == 200) { if (response.getBody() != null && response.getBody().getObject() != null) { - final JSONArray versions = response.getBody().getObject().getJSONArray("versions"); - final String latest = findLatestVersion(versions); // get the last version in the array - meta.setLatestVersion(latest); + final JSONArray versionsArray = response.getBody().getObject().getJSONArray("versions"); + final List versions = ((List)versionsArray.toList()).stream().map(Object::toString).toList(); + final String highestVersion = AbstractMetaAnalyzer.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); } return true; } else { @@ -125,23 +124,6 @@ private boolean performVersionCheck(final MetaModel meta, final Component compon return false; } - private String findLatestVersion(JSONArray versions) { - if (versions.length() < 1) { - return null; - } - - ComparableVersion latestVersion = new ComparableVersion(versions.getString(0)); - - for (int i = 1; i < versions.length(); i++) { - ComparableVersion version = new ComparableVersion(versions.getString(i)); - if (version.compareTo(latestVersion) > 0) { - latestVersion = version; - } - } - - return latestVersion.toString(); - } - private HttpResponse httpGet(String url) { final UnirestInstance ui = UnirestFactory.getUnirestInstance(); final HttpRequest request = ui.get(url).header("accept", "application/json"); diff --git a/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java index bddda44aad..bfbd6b4544 100644 --- a/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java +++ b/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java @@ -18,24 +18,27 @@ */ package org.dependencytrack.tasks.repositories; -import alpine.common.logging.Logger; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.dependencytrack.common.UnirestFactory; +import org.dependencytrack.model.Component; +import org.dependencytrack.model.RepositoryType; + import com.github.packageurl.PackageURL; + +import alpine.common.logging.Logger; import kong.unirest.GetRequest; import kong.unirest.HttpRequest; import kong.unirest.HttpResponse; import kong.unirest.JsonNode; import kong.unirest.UnirestException; import kong.unirest.UnirestInstance; -import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; -import org.dependencytrack.common.UnirestFactory; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.RepositoryType; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; /** * An IMetaAnalyzer implementation that supports Pypi. @@ -74,7 +77,7 @@ public MetaModel analyze(final Component component) { final UnirestInstance ui = UnirestFactory.getUnirestInstance(); final MetaModel meta = new MetaModel(component); if (component.getPurl() != null) { - final String url = String.format(baseUrl + API_URL, component.getPurl().getName()); + final String url = String.format(baseUrl.concat(API_URL), component.getPurl().getName()); try { final HttpRequest request = ui.get(url) .header("accept", "application/json"); @@ -85,26 +88,8 @@ public MetaModel analyze(final Component component) { if (response.getStatus() == 200) { if (response.getBody() != null && response.getBody().getObject() != null) { - final JSONObject info = response.getBody().getObject().getJSONObject("info"); - final String latest = info.optString("version", null); - if (latest != null) { - meta.setLatestVersion(latest); - final JSONObject releases = response.getBody().getObject().getJSONObject("releases"); - final JSONArray latestArray = releases.getJSONArray(latest); - if (latestArray.length() > 0) { - final JSONObject release = latestArray.getJSONObject(0); - final String updateTime = release.optString("upload_time", null); - if (updateTime != null) { - final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - try { - final Date published = dateFormat.parse(updateTime); - meta.setPublishedTimestamp(published); - } catch (ParseException e) { - LOGGER.warn("An error occurred while parsing upload time", e); - } - } - } - } + final JSONObject releases = response.getBody().getObject().getJSONObject("releases"); + analyzeReleases(meta, releases); } } else { handleUnexpectedHttpResponse(LOGGER, url, response.getStatus(), response.getStatusText(), component); @@ -115,4 +100,29 @@ public MetaModel analyze(final Component component) { } return meta; } + + private void analyzeReleases(final MetaModel meta, final JSONObject releases) { + List versions = new ArrayList<>(releases.keySet()); + final String highestVersion = AbstractMetaAnalyzer.findHighestVersion(versions); + meta.setLatestVersion(highestVersion); + if (highestVersion != null) { + final JSONObject release = releases.getJSONArray(highestVersion).getJSONObject(0); + final String updateTime = release.optString("upload_time", null); + if (updateTime != null) { + Date published = getPublishedTimestamp(updateTime); + meta.setPublishedTimestamp(published); + } + } + } + + private Date getPublishedTimestamp(final String updateTime) { + final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + Date published = null; + try { + published = dateFormat.parse(updateTime); + } catch (ParseException e) { + LOGGER.warn("An error occurred while parsing upload time", e); + } + return published; + } } diff --git a/src/test/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzerTest.java new file mode 100644 index 0000000000..6036d1eef2 --- /dev/null +++ b/src/test/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzerTest.java @@ -0,0 +1,116 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) Steve Springett. All Rights Reserved. + */ +package org.dependencytrack.tasks.repositories; + +import java.util.Arrays; + +import org.junit.Assert; +import org.junit.Test; + +public class AbstractMetaAnalyzerTest { + + @Test + public void testHighestVersion() { + Assert.assertNull(AbstractMetaAnalyzer.highestVersion(null, null)); + Assert.assertEquals("0.0.1", AbstractMetaAnalyzer.highestVersion("0.0.1", null)); + Assert.assertEquals("0.0.1", AbstractMetaAnalyzer.highestVersion(null, "0.0.1")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("2-SNAPSHOT", "2")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("2-RC1", "2")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("2-alpha", "2")); + Assert.assertEquals("2.0.0", AbstractMetaAnalyzer.highestVersion("1.9.9.9", "2.0.0")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("1.9", "2")); + Assert.assertEquals("0.1", AbstractMetaAnalyzer.highestVersion("0.1", "0.0.1")); + // N.B. we would like to see 2 being higher as 2-a, but since we're only interested in stable versions it doesn't maater + Assert.assertEquals("2-a", AbstractMetaAnalyzer.highestVersion("2-a", "2")); + Assert.assertEquals("2", AbstractMetaAnalyzer.highestVersion("v2", "2")); + Assert.assertEquals("v2.2", AbstractMetaAnalyzer.highestVersion("v2.1", "v2.2")); + Assert.assertEquals("v2.2", AbstractMetaAnalyzer.highestVersion("v2.2", "v2.1")); + Assert.assertEquals("v2", AbstractMetaAnalyzer.highestVersion("1", "v2")); + Assert.assertEquals("v2", AbstractMetaAnalyzer.highestVersion("v1-rc1", "v2")); + Assert.assertEquals("v2", AbstractMetaAnalyzer.highestVersion("v2-alpha", "v2")); + Assert.assertEquals("v2", AbstractMetaAnalyzer.highestVersion("v2-snapshot", "v2")); + Assert.assertEquals("v2-snapshot", AbstractMetaAnalyzer.highestVersion("v1", "v2-snapshot")); + } + + @Test + public void testIsStableVersion() { + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("version")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1.0")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1.0.0")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1.0.0")); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion("1a")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1-a")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1-alpha")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1-b")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1.1-snapshot")); + Assert.assertFalse(AbstractMetaAnalyzer.isStableVersion("1.2.3-rc1")); + } + + @Test + public void testfindHighestStableOrUnstableVersion() { + Assert.assertNull(AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList())); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList("1"))); + Assert.assertEquals("2-snapshot", AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList("1", "2-snapshot"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList("1", "2"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList("1", "2", "2-snapshot"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList("1", "2", "2-alpha"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList("1", "2", "2-beta"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableOrUnstableVersion(Arrays.asList("1", "2", "2-rc1"))); + } + + @Test + public void testfindHighestStableVersion() { + Assert.assertNull(AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList())); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "0.9.9"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2-snapshot"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2-rc1"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2-alpha"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2-m1"))); + Assert.assertEquals("2.0", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1.2.3", "2.0"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2", "2-snapshot"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2", "2-a"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2", "2-alpha"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2", "2-beta"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1", "2", "2-rc1"))); + Assert.assertNull(AbstractMetaAnalyzer.findHighestStableVersion(Arrays.asList("1-a", "2-snapshot", "2-rc1"))); + } + + @Test + public void testfindHighestVersion() { + Assert.assertNull(AbstractMetaAnalyzer.findHighestVersion(Arrays.asList())); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "0.9.9"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2-snapshot"))); + Assert.assertEquals("2-snapshot", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1-snapshot", "2-snapshot"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2-rc1"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2-alpha"))); + Assert.assertEquals("1", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2-m1"))); + Assert.assertEquals("2.0", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1.2.3", "2.0"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2", "2-snapshot"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2", "2-a"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2", "2-alpha"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2", "2-beta"))); + Assert.assertEquals("2", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1", "2", "2-rc1"))); + // N.B. 2.rc1 would be a better choise than 2-snapshot :-( + Assert.assertEquals("2-snapshot", AbstractMetaAnalyzer.findHighestVersion(Arrays.asList("1-a", "2-snapshot", "2-rc1"))); + } + +} diff --git a/src/test/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzerTest.java index 67f90a53dc..d5f9c6825d 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzerTest.java @@ -59,6 +59,8 @@ public void testAnalyzer() throws Exception { Assert.assertTrue(analyzer.isApplicable(component)); Assert.assertEquals(RepositoryType.COMPOSER, analyzer.supportedRepositoryType()); MetaModel metaModel = analyzer.analyze(component); + Assert.assertFalse(metaModel.getLatestVersion().startsWith("v")); + Assert.assertNotEquals("10.0.0", AbstractMetaAnalyzer.highestVersion("10.0.0", metaModel.getLatestVersion())); Assert.assertNotNull(metaModel.getLatestVersion()); Assert.assertNotNull(metaModel.getPublishedTimestamp()); } diff --git a/src/test/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzerTest.java index 2557a929af..f03dd9e3b9 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/HexMetaAnalyzerTest.java @@ -18,16 +18,17 @@ */ package org.dependencytrack.tasks.repositories; -import com.github.packageurl.PackageURL; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; import org.junit.Assert; import org.junit.Test; +import com.github.packageurl.PackageURL; + public class HexMetaAnalyzerTest { @Test - public void testAnalyzer() throws Exception { + public void testAnalyzerForPhoenix() throws Exception { Component component = new Component(); component.setPurl(new PackageURL("pkg:hex/phoenix@1.4.10")); @@ -35,6 +36,23 @@ public void testAnalyzer() throws Exception { Assert.assertTrue(analyzer.isApplicable(component)); Assert.assertEquals(RepositoryType.HEX, analyzer.supportedRepositoryType()); MetaModel metaModel = analyzer.analyze(component); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion(metaModel.getLatestVersion())); + Assert.assertNotEquals("1.6.15", AbstractMetaAnalyzer.highestVersion("1.6.15", metaModel.getLatestVersion())); + Assert.assertNotNull(metaModel.getLatestVersion()); + Assert.assertNotNull(metaModel.getPublishedTimestamp()); + } + + @Test + public void testAnalyzerForPartisan() throws Exception { + Component component = new Component(); + component.setPurl(new PackageURL("pkg:hex/partisan@1.4.10")); + + HexMetaAnalyzer analyzer = new HexMetaAnalyzer(); + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.HEX, analyzer.supportedRepositoryType()); + MetaModel metaModel = analyzer.analyze(component); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion(metaModel.getLatestVersion())); + Assert.assertNotEquals("4.0.9", AbstractMetaAnalyzer.highestVersion("4.0.9", metaModel.getLatestVersion())); Assert.assertNotNull(metaModel.getLatestVersion()); Assert.assertNotNull(metaModel.getPublishedTimestamp()); } diff --git a/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java index dd997d7d51..9266c280da 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/MavenMetaAnalyzerTest.java @@ -36,12 +36,9 @@ public void testAnalyzerForJUnit() throws Exception { Assert.assertTrue(analyzer.isApplicable(component)); Assert.assertEquals(RepositoryType.MAVEN, analyzer.supportedRepositoryType()); MetaModel metaModel = analyzer.analyze(component); - if (metaModel.getLatestVersion() == null) { - Assert.assertNull(metaModel.getPublishedTimestamp()); - } else { - Assert.assertFalse(metaModel.getLatestVersion().matches(MavenMetaAnalyzer.UNSTABLE_VERSIONS_PATTERN)); - // publishedTimestamp might be set or not - } + Assert.assertNotEquals("4.13.1", AbstractMetaAnalyzer.highestVersion("4.13.1", metaModel.getLatestVersion())); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion(metaModel.getLatestVersion())); + // publishedTimestamp might be set or not } @Test @@ -52,13 +49,10 @@ public void testAnalyzerForApacheCamel() throws Exception { MavenMetaAnalyzer analyzer = new MavenMetaAnalyzer(); Assert.assertTrue(analyzer.isApplicable(component)); Assert.assertEquals(RepositoryType.MAVEN, analyzer.supportedRepositoryType()); - MetaModel metaModel = analyzer.analyze(component); - if (metaModel.getLatestVersion() == null) { - Assert.assertNull(metaModel.getPublishedTimestamp()); - } else { - Assert.assertFalse(metaModel.getLatestVersion().matches(MavenMetaAnalyzer.UNSTABLE_VERSIONS_PATTERN)); - // publishedTimestamp might be set or not - } + MetaModel metaModel = analyzer.analyze(component); + Assert.assertNotEquals("3.20.1", AbstractMetaAnalyzer.highestVersion("3.20.1", metaModel.getLatestVersion())); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion(metaModel.getLatestVersion())); + // publishedTimestamp might be set or not } @Test @@ -73,11 +67,9 @@ public void testAnalyzerForScalaComponent() throws Exception { Assert.assertTrue(analyzer.isApplicable(component)); Assert.assertEquals(RepositoryType.MAVEN, analyzer.supportedRepositoryType()); MetaModel metaModel = analyzer.analyze(component); - if (metaModel.getLatestVersion() == null) { - Assert.assertNull(metaModel.getPublishedTimestamp()); - } else { - Assert.assertFalse(metaModel.getLatestVersion().matches(MavenMetaAnalyzer.UNSTABLE_VERSIONS_PATTERN)); - // publishedTimestamp might be set or not - } + Assert.assertNotEquals("2.6.0", AbstractMetaAnalyzer.highestVersion("2.6.0", metaModel.getLatestVersion())); + Assert.assertTrue(AbstractMetaAnalyzer.isStableVersion(metaModel.getLatestVersion())); + // publishedTimestamp might be set or not } + } diff --git a/src/test/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzerTest.java index a16856309f..3ef0718cb6 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/NpmMetaAnalyzerTest.java @@ -36,6 +36,6 @@ public void testAnalyzer() throws Exception { Assert.assertEquals(RepositoryType.NPM, analyzer.supportedRepositoryType()); MetaModel metaModel = analyzer.analyze(component); Assert.assertNotNull(metaModel.getLatestVersion()); - //Assert.assertNotNull(metaModel.getPublishedTimestamp()); // todo: not yet supported + Assert.assertNull(metaModel.getPublishedTimestamp()); // NPM does not register publication dates } } diff --git a/src/test/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzerTest.java index 4082d52dbc..a991881c2a 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/NugetMetaAnalyzerTest.java @@ -64,6 +64,26 @@ public void testAnalyzer() throws Exception { Assert.assertNotNull(metaModel.getPublishedTimestamp()); } + + @Test + public void testAnalyzerMicrosoftGraph() throws Exception { + Component component = new Component(); + component.setPurl(new PackageURL("pkg:nuget/Microsoft.Graph@3.35.0")); + + NugetMetaAnalyzer analyzer = new NugetMetaAnalyzer(); + + analyzer.setRepositoryBaseUrl("https://api.nuget.org"); + MetaModel metaModel = analyzer.analyze(component); + + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.NUGET, analyzer.supportedRepositoryType()); + Assert.assertFalse(metaModel.getLatestVersion().startsWith("5.0.0-preview")); + Assert.assertFalse(metaModel.getLatestVersion().startsWith("5.0.0-rc")); + Assert.assertNotEquals("4.0.0", AbstractMetaAnalyzer.highestVersion("4.0.0",metaModel.getLatestVersion())); + Assert.assertNotNull(metaModel.getLatestVersion()); + Assert.assertNotNull(metaModel.getPublishedTimestamp()); + } + @Test public void testAnalyzerWithPrivatePackageRepository() throws Exception { String mockIndexResponse = readResourceFileToString("/unit/tasks/repositories/https---localhost-1080-v3-index.json"); diff --git a/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java index 601a8841c1..d1e5fedb4f 100644 --- a/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java +++ b/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java @@ -18,16 +18,17 @@ */ package org.dependencytrack.tasks.repositories; -import com.github.packageurl.PackageURL; import org.dependencytrack.model.Component; import org.dependencytrack.model.RepositoryType; import org.junit.Assert; import org.junit.Test; +import com.github.packageurl.PackageURL; + public class PypiMetaAnalyzerTest { @Test - public void testAnalyzer() throws Exception { + public void testAnalyzerForFlask() throws Exception { Component component = new Component(); component.setPurl(new PackageURL("pkg:pypi/Flask@1.0.0")); @@ -35,6 +36,21 @@ public void testAnalyzer() throws Exception { Assert.assertTrue(analyzer.isApplicable(component)); Assert.assertEquals(RepositoryType.PYPI, analyzer.supportedRepositoryType()); MetaModel metaModel = analyzer.analyze(component); + Assert.assertNotEquals("2.2.2", AbstractMetaAnalyzer.highestVersion("2.2.2", metaModel.getLatestVersion())); + Assert.assertNotNull(metaModel.getLatestVersion()); + Assert.assertNotNull(metaModel.getPublishedTimestamp()); + } + + @Test + public void testAnalyzerForNumPi() throws Exception { + Component component = new Component(); + component.setPurl(new PackageURL("pkg:pypi/NumPi@1.0.0")); + + PypiMetaAnalyzer analyzer = new PypiMetaAnalyzer(); + Assert.assertTrue(analyzer.isApplicable(component)); + Assert.assertEquals(RepositoryType.PYPI, analyzer.supportedRepositoryType()); + MetaModel metaModel = analyzer.analyze(component); + Assert.assertNotEquals("0.3.0", AbstractMetaAnalyzer.highestVersion("0.3.0", metaModel.getLatestVersion())); Assert.assertNotNull(metaModel.getLatestVersion()); Assert.assertNotNull(metaModel.getPublishedTimestamp()); }