From df3bd05ee39d7affa3742545496c79a5e5c0f115 Mon Sep 17 00:00:00 2001 From: Przemyslaw Czuj Date: Thu, 18 May 2023 23:34:15 +0200 Subject: [PATCH] JPERF-1106: Gather Apache2 logs --- CHANGELOG.md | 15 ++++ .../awsinfrastructure/SuppressingRunnable.kt | 26 ++++++ .../api/jira/DataCenterFormula.kt | 22 +++-- .../tools/awsinfrastructure/api/jira/Jira.kt | 90 +++++++++++++++++-- .../api/jira/StandaloneFormula.kt | 17 ++-- .../awsinfrastructure/api/jira/StartedNode.kt | 5 +- .../api/jira/UriJiraFormula.kt | 34 ++++--- .../loadbalancer/ApacheProxyLoadBalancer.kt | 14 ++- .../loadbalancer/DiagnosableLoadBalancer.kt | 8 ++ .../loadbalancer/ProvisionedLoadBalancer.kt | 2 +- .../LoadBalancerDiagnosticsSource.kt | 23 +++++ .../SuppressingRunnableTest.kt | 69 ++++++++++++++ 12 files changed, 275 insertions(+), 50 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnable.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerDiagnosticsSource.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnableTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 9172d18b..b227cbb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,8 +23,23 @@ Dropping a requirement of a major version of a dependency is a new contract. ## [Unreleased] [Unreleased]: https://github.com/atlassian/aws-infrastructure/compare/release-2.29.0...master +## Added +- Add `DiagnosableLoadBalancer`. +- Add `Jira.Builder`. +- Add ability to `gatherDiagnostics` from `ApacheProxyLoadBalancer` by making it `DiagnosableLoadBalancer`. +- Make it possible to specify `extraResults` inside `Jira`, so that it's possible to `gatherResults` from any of Jira hosted integrations, e.g. load balancer or plugins. +- Let `Jira` produced by `DataCenterFormula` gather logs of `ApacheProxyLoadBalancer`. Resolve [JPERF-1106]. +- Make `StartedNode` a `MeasurementSource`. + +## Deprecated +- Deprecate `Jira` constructors in favor of `Jira.Builder`. + ### Fixed - Change default virtual user instance type to `c5.9xlarge`. It's better, cheaper and there seem to be availability issues with previous default (`c4.8xlarge`). +- Let every `MeasurementSource` inside `Jira` finish its gathering even if one of them fails. Fix [JPERF-1114]. + +[JPERF-1106]: https://ecosystem.atlassian.net/browse/JPERF-1106 +[JPERF-1114]: https://ecosystem.atlassian.net/browse/JPERF-1114 ## [2.29.0] - 2023-03-24 [2.29.0]: https://github.com/atlassian/aws-infrastructure/compare/release-2.28.0...release-2.29.0 diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnable.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnable.kt new file mode 100644 index 00000000..dd617fb0 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnable.kt @@ -0,0 +1,26 @@ +package com.atlassian.performance.tools.awsinfrastructure + +internal class SuppressingRunnable( + private val delegates: Iterable +) : Runnable { + override fun run() { + val exceptions = delegates.fold(emptyList()) { exceptions, runnable -> + val exception = try { + runnable.run() + null + } catch (e: Exception) { + e + } + exceptions + listOfNotNull(exception) + } + + when { + exceptions.isEmpty() -> return + exceptions.size == 1 -> throw exceptions[0] + else -> { + val root = Exception("Multiple exceptions were thrown and are added suppressed into this one") + throw exceptions.fold(root) { root, it -> root.addSuppressed(it); root } + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt index 06d344b4..a67187a8 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt @@ -12,6 +12,7 @@ import com.atlassian.performance.tools.awsinfrastructure.api.hardware.M4ExtraLar import com.atlassian.performance.tools.awsinfrastructure.api.hardware.Volume import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheEc2LoadBalancerFormula import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheProxyLoadBalancer +import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.LoadBalancerFormula import com.atlassian.performance.tools.awsinfrastructure.api.network.Network import com.atlassian.performance.tools.awsinfrastructure.api.network.NetworkFormula @@ -20,6 +21,7 @@ import com.atlassian.performance.tools.awsinfrastructure.jira.DataCenterNodeForm import com.atlassian.performance.tools.awsinfrastructure.jira.DiagnosableNodeFormula import com.atlassian.performance.tools.awsinfrastructure.jira.StandaloneNodeFormula import com.atlassian.performance.tools.awsinfrastructure.jira.home.SharedHomeFormula +import com.atlassian.performance.tools.awsinfrastructure.loadbalancer.LoadBalancerDiagnosticsSource.Extension.asMeasurementSource import com.atlassian.performance.tools.concurrency.api.AbruptExecutorService import com.atlassian.performance.tools.concurrency.api.submitWithLogContext import com.atlassian.performance.tools.infrastructure.api.app.Apps @@ -382,16 +384,18 @@ class DataCenterFormula private constructor( loadBalancer.waitUntilHealthy(Duration.ofMinutes(5)) } - val jira = Jira( - nodes = nodes, - jiraHome = RemoteLocation( - sharedHomeSsh.host, - sharedHome.get().remoteSharedHome - ), - database = databaseDataLocation, - address = loadBalancer.uri, - jmxClients = jiraNodes.mapIndexed { i, node -> configs[i].remoteJmx.getClient(node.publicIpAddress) } + val jiraHomeLocation = RemoteLocation( + sharedHomeSsh.host, + sharedHome.get().remoteSharedHome ) + val loadBalancerResultsSource = (provisionedLoadBalancer.loadBalancer as? DiagnosableLoadBalancer) + ?.asMeasurementSource(resultsTransport.location) + val jira = Jira.Builder(address = loadBalancer.uri, jiraHome = jiraHomeLocation) + .database(databaseDataLocation) + .nodes(nodes) + .jmxClients(jiraNodes.mapIndexed { i, node -> configs[i].remoteJmx.getClient(node.publicIpAddress) }) + .extraResults(listOfNotNull(loadBalancerResultsSource)) + .build() logger.info("$jira is set up, will expire ${jiraStack.expiry}") return ProvisionedJira.Builder(jira) .resource( diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt index de9c6ba5..f5f25ced 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt @@ -1,41 +1,113 @@ package com.atlassian.performance.tools.awsinfrastructure.api.jira +import com.atlassian.performance.tools.awsinfrastructure.SuppressingRunnable import com.atlassian.performance.tools.awsinfrastructure.api.RemoteLocation import com.atlassian.performance.tools.concurrency.api.submitWithLogContext import com.atlassian.performance.tools.infrastructure.api.MeasurementSource import com.atlassian.performance.tools.infrastructure.api.jvm.jmx.JmxClient -import com.atlassian.performance.tools.jvmtasks.api.TaskTimer.time import com.google.common.util.concurrent.ThreadFactoryBuilder import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import java.net.URI import java.util.concurrent.Executors -class Jira( +/** + * @param extraResultsSources source of results/diagnostics of: reverse proxy, Crowd, LDAP, DVCS, DB, Jira plugins + * or anything that can be integrated with Jira as part of web application provisioning + */ +class Jira private constructor( private val nodes: List, val jiraHome: RemoteLocation, val database: RemoteLocation?, val address: URI, - val jmxClients: List = emptyList() + val jmxClients: List, + private val extraResultsSources: List ) : MeasurementSource { private val logger: Logger = LogManager.getLogger(this::class.java) + @Deprecated("Use Jira.Builder instead.") + constructor( + nodes: List, + jiraHome: RemoteLocation, + database: RemoteLocation?, + address: URI, + jmxClients: List + ) : this( + nodes = nodes, + jiraHome = jiraHome, + database = database, + address = address, + jmxClients = jmxClients, + extraResultsSources = emptyList() + ) + + @Deprecated("Use Jira.Builder instead.") + constructor( + nodes: List, + jiraHome: RemoteLocation, + database: RemoteLocation?, + address: URI + ) : this( + nodes = nodes, + jiraHome = jiraHome, + database = database, + address = address, + jmxClients = emptyList(), + extraResultsSources = emptyList() + ) + override fun gatherResults() { - if (nodes.isEmpty()) { - logger.warn("No Jira nodes known to JPT, not downloading node results") + val firstNode = nodes.firstOrNull() + val measurementSources = extraResultsSources + nodes + listOfNotNull(firstNode?.let { AnalyticsLogsSource(it) }) + if (measurementSources.isEmpty()) { + logger.warn("No result sources known to Jira, can't download anything") return } val executor = Executors.newFixedThreadPool( - nodes.size.coerceAtMost(4), + measurementSources.size.coerceAtMost(4), ThreadFactoryBuilder() .setNameFormat("results-gathering-thread-%d") .build() ) - nodes.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } } - .forEach { it.get() } - time("gather analytics") { nodes.firstOrNull()?.gatherAnalyticLogs() } + SuppressingRunnable( + measurementSources.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } } + .map { Runnable { it.get() } } + ).run() + executor.shutdownNow() } override fun toString() = "Jira(address=$address)" + + class Builder( + private val address: URI, + private val jiraHome: RemoteLocation + ) { + private var database: RemoteLocation? = null + private var nodes: List = emptyList() + private var jmxClients: List = emptyList() + private var extraResults: List = emptyList() + + fun database(database: RemoteLocation?) = apply { this.database = database } + fun nodes(nodes: List) = apply { this.nodes = nodes } + fun jmxClients(jmxClients: List) = apply { this.jmxClients = jmxClients } + fun extraResults(extraResults: List) = apply { this.extraResults = extraResults } + + fun build() = Jira( + nodes = nodes, + jiraHome = jiraHome, + database = database, + address = address, + jmxClients = jmxClients, + extraResultsSources = extraResults + ) + } + + private class AnalyticsLogsSource( + private val node: StartedNode + ) : MeasurementSource { + override fun gatherResults() { + node.gatherAnalyticLogs() + } + } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StandaloneFormula.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StandaloneFormula.kt index b116e60e..ca58c7ea 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StandaloneFormula.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StandaloneFormula.kt @@ -291,16 +291,15 @@ class StandaloneFormula private constructor( logger.warn("It's possible that defined external access to Jira resources (e.g. http, debug, splunk) wasn't granted.") } - val jira = Jira( - nodes = listOf(node), - jiraHome = RemoteLocation( - jiraSsh.host, - provisionedNode.jiraHome - ), - database = databaseDataLocation, - address = jiraPublicHttpAddress, - jmxClients = listOf(config.remoteJmx.getClient(jiraPublicIp)) + val jiraHomeLocation = RemoteLocation( + host = jiraSsh.host, + location = provisionedNode.jiraHome ) + val jira = Jira.Builder(address = jiraPublicHttpAddress, jiraHome = jiraHomeLocation) + .nodes(listOf(node)) + .database(databaseDataLocation) + .jmxClients(listOf(config.remoteJmx.getClient(jiraPublicIp))) + .build() logger.info("$jira is set up, will expire ${jiraStack.expiry}") return ProvisionedJira.Builder(jira) .resource( diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt index 52843d74..29343249 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.awsinfrastructure.api.jira import com.atlassian.performance.tools.aws.api.Storage import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli +import com.atlassian.performance.tools.infrastructure.api.MeasurementSource import com.atlassian.performance.tools.infrastructure.api.jira.JiraGcLog import com.atlassian.performance.tools.infrastructure.api.process.RemoteMonitoringProcess import com.atlassian.performance.tools.ssh.api.Ssh @@ -15,10 +16,10 @@ class StartedNode( private val unpackedProduct: String, private val monitoringProcesses: List, private val ssh: Ssh -) { +) : MeasurementSource { private val resultsDirectory = "results" - fun gatherResults() { + override fun gatherResults() { ssh.newConnection().use { shell -> monitoringProcesses.forEach { it.stop(shell) } val nodeResultsDirectory = "$resultsDirectory/'$name'" diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/UriJiraFormula.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/UriJiraFormula.kt index 879596c5..3f7b0317 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/UriJiraFormula.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/UriJiraFormula.kt @@ -1,6 +1,9 @@ package com.atlassian.performance.tools.awsinfrastructure.api.jira -import com.atlassian.performance.tools.aws.api.* +import com.atlassian.performance.tools.aws.api.Aws +import com.atlassian.performance.tools.aws.api.Investment +import com.atlassian.performance.tools.aws.api.SshKey +import com.atlassian.performance.tools.aws.api.Storage import com.atlassian.performance.tools.awsinfrastructure.api.RemoteLocation import com.atlassian.performance.tools.ssh.api.SshHost import com.atlassian.performance.tools.ssh.api.auth.PasswordAuthentication @@ -18,22 +21,17 @@ class UriJiraFormula( key: Future, roleProfile: String, aws: Aws - ): ProvisionedJira = ProvisionedJira - .Builder( - Jira( - nodes = emptyList(), - jiraHome = RemoteLocation( - host = SshHost( - ipAddress = "unknown", - userName = "unknown", - authentication = PasswordAuthentication("unknown"), - port = -1 - ), - location = "unknown" - ), - database = null, - address = jiraAddress - ) + ): ProvisionedJira { + val jiraHomeLocation = RemoteLocation( + host = SshHost( + ipAddress = "unknown", + userName = "unknown", + authentication = PasswordAuthentication("unknown"), + port = -1 + ), + location = "unknown" ) - .build() + val jira = Jira.Builder(jiraAddress, jiraHomeLocation).build() + return ProvisionedJira.Builder(jira).build() + } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt index d6d5957b..4521a14b 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt @@ -1,7 +1,8 @@ package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer +import com.atlassian.performance.tools.aws.api.StorageLocation +import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli import com.atlassian.performance.tools.infrastructure.api.Sed -import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.ssh.api.Ssh @@ -14,7 +15,7 @@ class ApacheProxyLoadBalancer private constructor( private val ssh: Ssh, ipAddress: String, httpPort: Int -) : LoadBalancer { +) : DiagnosableLoadBalancer { @Deprecated(message = "Use ApacheProxyLoadBalancer.Builder instead.") constructor( @@ -93,6 +94,15 @@ class ApacheProxyLoadBalancer private constructor( connection.execute("echo \"$line\" | sudo tee -a $APACHE_CONFIG_PATH") } + override fun gatherDiagnostics(location: StorageLocation) { + ssh.newConnection().use { connection -> + val resultsDir = "/tmp/s3-results" + connection.execute("mkdir -p $resultsDir") + connection.execute("cp -R /var/log/apache2 $resultsDir") + AwsCli().upload(location, connection, resultsDir, Duration.ofMinutes(1)) + } + } + class Builder( private val ssh: Ssh ) { diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt new file mode 100644 index 00000000..e0e2a040 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt @@ -0,0 +1,8 @@ +package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer + +import com.atlassian.performance.tools.aws.api.StorageLocation +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer + +interface DiagnosableLoadBalancer : LoadBalancer { + fun gatherDiagnostics(location: StorageLocation) +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ProvisionedLoadBalancer.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ProvisionedLoadBalancer.kt index fe110e17..41a864c9 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ProvisionedLoadBalancer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ProvisionedLoadBalancer.kt @@ -4,8 +4,8 @@ import com.atlassian.performance.tools.aws.api.Resource import com.atlassian.performance.tools.aws.api.UnallocatedResource import com.atlassian.performance.tools.awsinfrastructure.api.network.access.AccessProvider import com.atlassian.performance.tools.awsinfrastructure.api.network.access.AccessRequester -import com.atlassian.performance.tools.awsinfrastructure.api.network.access.NoAccessRequester import com.atlassian.performance.tools.awsinfrastructure.api.network.access.NoAccessProvider +import com.atlassian.performance.tools.awsinfrastructure.api.network.access.NoAccessRequester import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer class ProvisionedLoadBalancer private constructor( diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerDiagnosticsSource.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerDiagnosticsSource.kt new file mode 100644 index 00000000..0d5fe94a --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerDiagnosticsSource.kt @@ -0,0 +1,23 @@ +package com.atlassian.performance.tools.awsinfrastructure.loadbalancer + +import com.atlassian.performance.tools.aws.api.StorageLocation +import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer +import com.atlassian.performance.tools.infrastructure.api.MeasurementSource + +internal class LoadBalancerDiagnosticsSource( + private val loadBalancer: DiagnosableLoadBalancer, + private val location: StorageLocation +) : MeasurementSource { + internal object Extension { + fun DiagnosableLoadBalancer.asMeasurementSource( + location: StorageLocation + ) = LoadBalancerDiagnosticsSource( + loadBalancer = this, + location = location + ) + } + + override fun gatherResults() { + loadBalancer.gatherDiagnostics(location) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnableTest.kt b/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnableTest.kt new file mode 100644 index 00000000..1580a3fd --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/SuppressingRunnableTest.kt @@ -0,0 +1,69 @@ +package com.atlassian.performance.tools.awsinfrastructure + +import org.hamcrest.CoreMatchers.* +import org.junit.Assert.* +import org.junit.Test + +class SuppressingRunnableTest { + @Test + fun shouldExecuteAllEvenIfFirstFails() { + var executed1 = false + var executed2 = false + var executed3 = false + val runnable = SuppressingRunnable( + listOf( + Runnable { executed1 = true; throw Exception("Fail 1") }, + Runnable { executed2 = true; throw Exception("Fail 2") }, + Runnable { executed3 = true; throw Exception("Fail 3") } + ) + ) + + try { + runnable.run() + } catch (e: Exception) { + // Expected and ignored, so that we can go to asserts + } + + assertTrue(executed1) + assertTrue(executed2) + assertTrue(executed3) + } + + @Test + fun shouldThrowAllFailures() { + val runnable = SuppressingRunnable( + listOf( + Runnable { throw Exception("Banana") }, + Runnable { throw Exception("Apple") }, + Runnable { throw Exception("Pear") }, + Runnable { throw Exception("Peach") } + ) + ) + + val exception = try { + runnable.run() + null + } catch (e: Exception) { + e + } + + assertNotNull(exception) + val allExceptions = listOf(exception!!) + exception.suppressed.toList() + val allMessages = allExceptions.map { it.message } + assertThat(allMessages, hasItems("Banana", "Apple", "Pear", "Peach")) + } + + + @Test + fun shouldExecuteAll() { + val allIndexes = Array(20) { it } + val finishedIndexes = mutableListOf() + val runnable = SuppressingRunnable( + allIndexes.map { index -> Runnable { finishedIndexes.add(index) } } + ) + + runnable.run() + + assertThat(finishedIndexes, hasItems(*allIndexes)) + } +} \ No newline at end of file