Improvements to the performance testing harness (#3484)

perf-tests/README.md

@@ -20,21 +20,18 @@ which displays help similar to:
 
 ```
 [error] Usage: perf [options]
-[error]   -s, --server <value>    Comma-separated list of short server names, or '*' for all. Available servers: http4s.TapirMulti, http4s.Tapir, http4s.VanillaMulti, http4s.Vanilla, 
-netty.cats.TapirMulti, netty.cats.Tapir, netty.future.TapirMulti, netty.future.Tapir, pekko.TapirMulti, pekko.Tapir, pekko.VanillaMulti, pekko.Vanilla, play.TapirMulti, play.Tapir,
-play.VanillaMulti, play.Vanilla, vertx.TapirMulti, vertx.Tapir, vertx.VanillaMulti, vertx.Vanilla, vertx.cats.TapirMulti, vertx.cats.Tapir
-[error]   -m, --sim <value>       Comma-separated list of short simulation names, or '*' for all. Available simulations: PostBytes, PostFile, PostLongBytes, PostLongString, 
-PostString, SimpleGetMultiRoute, SimpleGet
+[error]   -s, --server <value>    Comma-separated list of short server names, or groups like 'netty.*', 'pekko.*', etc. Available servers: http4s.TapirInterceptorMulti, http4s.TapirMulti, http4s.Tapir, http4s.VanillaMulti, http4s.Vanilla, netty.cats.TapirInterceptorMulti, netty.cats.TapirMulti, netty.cats.Tapir, netty.future.TapirInterceptorMulti, netty.future.TapirMulti, netty.future.Tapir, pekko.TapirInterceptorMulti, pekko.TapirMulti, pekko.Tapir, pekko.VanillaMulti, pekko.Vanilla, play.TapirInterceptorMulti, play.TapirMulti, play.Tapir, play.VanillaMulti, play.Vanilla, vertx.TapirInterceptorMulti, vertx.TapirMulti, vertx.Tapir, vertx.VanillaMulti, vertx.Vanilla, vertx.cats.TapirInterceptorMulti, vertx.cats.TapirMulti, vertx.cats.Tapir
+[error]   -m, --sim <value>       Comma-separated list of short simulation names, or '*' for all. Available simulations: PostBytes, PostFile, PostLongBytes, PostLongString, PostString, SimpleGetMultiRoute, SimpleGet
 [error]   -u, --users <value>     Number of concurrent users, default is 1
 [error]   -d, --duration <value>  Single simulation duration in seconds, default is 10
 [error]   -g, --gatling-reports   Generate Gatling reports for individuals sims, may significantly affect total time (disabled by default)
 ```
 
 ## Examples
 
-1. Run all sims on all servers with other options set to default (Careful, may take quite some time!):
+1. Run all sims on all pekko-http servers with other options set to default:
 ```
-perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner -s * -m *
+perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner -s pekko.* -m *
 ```
 
 2. Run all sims on http4s servers, with each simulation running for 5 seconds:

perf-tests/src/main/scala/sttp/tapir/perf/Common.scala

@@ -6,6 +6,7 @@ import java.util.Date
 
 import scala.concurrent.duration._
 import scala.util.Random
+import sttp.tapir.server.interceptor.CustomiseInterceptors
 
 object Common {
   val rootPackage = "sttp.tapir.perf"
@@ -15,5 +16,9 @@ object Common {
   val Port = 8080
   val TmpDir: File = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile
   def newTempFilePath(): Path = TmpDir.toPath.resolve(s"tapir-${new Date().getTime}-${Random.nextLong()}")
-
+  def buildOptions[F[_], O](customiseInterceptors: CustomiseInterceptors[F, O],  withServerLog: Boolean): O = 
+    (if (withServerLog == false)
+      customiseInterceptors.serverLog(None)
+    else
+      customiseInterceptors).options
 }

perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala

@@ -50,17 +50,14 @@ object Tapir extends Endpoints {
 
   implicit val mErr: MonadError[IO] = new CatsMonadError[IO]
 
-  val serverOptions = Http4sServerOptions
-    .customiseInterceptors[IO]
-    .serverLog(None)
-    .options
-
-  val router: Int => HttpRoutes[IO] = (nRoutes: Int) =>
+  def router(nRoutes: Int, withServerLog: Boolean = false): HttpRoutes[IO] = {
+    val serverOptions = buildOptions(Http4sServerOptions.customiseInterceptors[IO], withServerLog)
     Router("/" -> {
       Http4sServerInterpreter[IO](serverOptions).toRoutes(
         genEndpointsIO(nRoutes)
       )
     })
+  }
 }
 
 object server {
@@ -78,5 +75,6 @@ object server {
 
 object TapirServer extends ServerRunner { override def start = server.runServer(Tapir.router(1)) }
 object TapirMultiServer extends ServerRunner { override def start = server.runServer(Tapir.router(128)) }
+object TapirInterceptorMultiServer extends ServerRunner { override def start = server.runServer(Tapir.router(128, withServerLog = true)) }
 object VanillaServer extends ServerRunner { override def start = server.runServer(Vanilla.router(1)) }
 object VanillaMultiServer extends ServerRunner { override def start = server.runServer(Vanilla.router(128)) }

perf-tests/src/main/scala/sttp/tapir/perf/netty/cats/NettyCats.scala

@@ -13,12 +13,12 @@ object Tapir extends Endpoints
 
 object NettyCats {
 
-  def runServer(endpoints: List[ServerEndpoint[Any, IO]]): IO[ServerRunner.KillSwitch] = {
+  def runServer(endpoints: List[ServerEndpoint[Any, IO]], withServerLog: Boolean = false): IO[ServerRunner.KillSwitch] = {
     val declaredPort = Port
     val declaredHost = "0.0.0.0"
     (for {
       dispatcher <- Dispatcher.parallel[IO]
-      serverOptions = NettyCatsServerOptions.customiseInterceptors(dispatcher).serverLog(None).options
+      serverOptions = buildOptions(NettyCatsServerOptions.customiseInterceptors(dispatcher), withServerLog)
       server <- NettyCatsServer.io()
       _ <-
         Resource.make(
@@ -34,3 +34,6 @@ object NettyCats {
 
 object TapirServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpointsIO(1)) }
 object TapirMultiServer extends ServerRunner { override def start = NettyCats.runServer(Tapir.genEndpointsIO(128)) }
+object TapirInterceptorMultiServer extends ServerRunner {
+  override def start = NettyCats.runServer(Tapir.genEndpointsIO(128), withServerLog = true)
+}

perf-tests/src/main/scala/sttp/tapir/perf/netty/future/NettyFuture.scala

@@ -14,10 +14,10 @@ object Tapir extends Endpoints
 
 object NettyFuture {
 
-  def runServer(endpoints: List[ServerEndpoint[Any, Future]]): IO[ServerRunner.KillSwitch] = {
+  def runServer(endpoints: List[ServerEndpoint[Any, Future]], withServerLog: Boolean = false): IO[ServerRunner.KillSwitch] = {
     val declaredPort = Port
     val declaredHost = "0.0.0.0"
-    val serverOptions = NettyFutureServerOptions.customiseInterceptors.serverLog(None).options
+    val serverOptions = buildOptions(NettyFutureServerOptions.customiseInterceptors, withServerLog)
     // Starting netty server
     val serverBinding: IO[NettyFutureServerBinding] =
       IO.fromFuture(
@@ -36,3 +36,4 @@ object NettyFuture {
 
 object TapirServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpointsFuture(1)) }
 object TapirMultiServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpointsFuture(128)) }
+object TapirInterceptorMultiServer extends ServerRunner { override def start = NettyFuture.runServer(Tapir.genEndpointsFuture(128), withServerLog = true) }

perf-tests/src/main/scala/sttp/tapir/perf/pekko/PekkoHttp.scala

@@ -58,16 +58,12 @@ object Vanilla {
 }
 
 object Tapir extends Endpoints {
-  val serverOptions = PekkoHttpServerOptions
-    .customiseInterceptors(ExecutionContext.Implicits.global)
-    .serverLog(None)
-    .options
-
-  def router: Int => ActorSystem => Route = (nRoutes: Int) =>
-    (actorSystem: ActorSystem) =>
-      PekkoHttpServerInterpreter(serverOptions)(actorSystem.dispatcher).toRoute(
-        genEndpointsFuture(nRoutes)
-      )
+  def router(nRoutes: Int, withServerLog: Boolean = false): ActorSystem => Route = { (actorSystem: ActorSystem) =>
+    val serverOptions = buildOptions(PekkoHttpServerOptions.customiseInterceptors(ExecutionContext.Implicits.global), withServerLog)
+    PekkoHttpServerInterpreter(serverOptions)(actorSystem.dispatcher).toRoute(
+      genEndpointsFuture(nRoutes)
+    )
+  }
 }
 
 object PekkoHttp {
@@ -90,5 +86,8 @@ object PekkoHttp {
 
 object TapirServer extends ServerRunner { override def start = PekkoHttp.runServer(Tapir.router(1)) }
 object TapirMultiServer extends ServerRunner { override def start = PekkoHttp.runServer(Tapir.router(128)) }
+object TapirInterceptorMultiServer extends ServerRunner {
+  override def start = PekkoHttp.runServer(Tapir.router(128, withServerLog = true))
+}
 object VanillaServer extends ServerRunner { override def start = PekkoHttp.runServer(Vanilla.router(1)) }
 object VanillaMultiServer extends ServerRunner { override def start = PekkoHttp.runServer(Vanilla.router(128)) }

perf-tests/src/main/scala/sttp/tapir/perf/play/Play.scala

@@ -74,11 +74,11 @@ object Vanilla extends ControllerHelpers {
 }
 
 object Tapir extends Endpoints {
-  val router: Int => ActorSystem => Routes = (nRoutes: Int) =>
+  def router(nRoutes: Int, withServerLog: Boolean = false): ActorSystem => Routes =
     (actorSystem: ActorSystem) => {
       implicit val actorSystemForMaterializer: ActorSystem = actorSystem
       implicit val ec: ExecutionContext = actorSystem.dispatcher
-      val serverOptions = PlayServerOptions.customiseInterceptors().serverLog(None).options
+      val serverOptions = buildOptions(PlayServerOptions.customiseInterceptors(), withServerLog)
       PlayServerInterpreter(serverOptions).toRoutes(
         genEndpointsFuture(nRoutes)
       )
@@ -109,5 +109,6 @@ object Play {
 
 object TapirServer extends ServerRunner { override def start = Play.runServer(Tapir.router(1)) }
 object TapirMultiServer extends ServerRunner { override def start = Play.runServer(Tapir.router(128)) }
+object TapirInterceptorMultiServer extends ServerRunner { override def start = Play.runServer(Tapir.router(128, withServerLog = true)) }
 object VanillaServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(1)) }
 object VanillaMultiServer extends ServerRunner { override def start = Play.runServer(Vanilla.router(128)) }

perf-tests/src/main/scala/sttp/tapir/perf/vertx/Vertx.scala

@@ -12,8 +12,8 @@ import sttp.tapir.server.vertx.VertxFutureServerInterpreter
 import sttp.tapir.server.vertx.VertxFutureServerOptions
 
 object Tapir extends Endpoints {
-  def route: Int => Router => Route = { (nRoutes: Int) => router =>
-    val serverOptions = VertxFutureServerOptions.customiseInterceptors.serverLog(None).options
+  def route(nRoutes: Int, withServerLog: Boolean = false): Router => Route = { router =>
+    val serverOptions = buildOptions(VertxFutureServerOptions.customiseInterceptors, withServerLog)
     val interpreter = VertxFutureServerInterpreter(serverOptions)
     genEndpointsFuture(nRoutes).map(interpreter.route(_)(router)).last
   }
@@ -96,6 +96,9 @@ object VertxRunner {
 }
 
 object TapirServer extends ServerRunner { override def start = VertxRunner.runServer(Tapir.route(1)) }
-object TapirMultiServer extends ServerRunner { override def start = VertxRunner.runServer(Tapir.route(127)) }
+object TapirMultiServer extends ServerRunner { override def start = VertxRunner.runServer(Tapir.route(128)) }
+object TapirInterceptorMultiServer extends ServerRunner {
+  override def start = VertxRunner.runServer(Tapir.route(128, withServerLog = true))
+}
 object VanillaServer extends ServerRunner { override def start = VertxRunner.runServer(Vanilla.route(1)) }
-object VanillaMultiServer extends ServerRunner { override def start = VertxRunner.runServer(Vanilla.route(127)) }
+object VanillaMultiServer extends ServerRunner { override def start = VertxRunner.runServer(Vanilla.route(128)) }

perf-tests/src/main/scala/sttp/tapir/perf/vertx/cats/VertxCats.scala

@@ -4,26 +4,28 @@ import cats.effect.IO
 import cats.effect.std.Dispatcher
 import io.vertx.ext.web.Route
 import io.vertx.ext.web.Router
+import sttp.tapir.perf.Common._
 import sttp.tapir.perf.apis.{Endpoints, ServerRunner}
 import sttp.tapir.perf.vertx.VertxRunner
 import sttp.tapir.server.vertx.cats.VertxCatsServerInterpreter
 import sttp.tapir.server.vertx.cats.VertxCatsServerOptions
 
 object Tapir extends Endpoints {
-  def route(dispatcher: Dispatcher[IO]): Int => Router => Route = { (nRoutes: Int) => router =>
-    val serverOptions = VertxCatsServerOptions.customiseInterceptors[IO](dispatcher).serverLog(None).options
+  def route(dispatcher: Dispatcher[IO], withServerLog: Boolean): Int => Router => Route = { (nRoutes: Int) => (router: Router) =>
+    val serverOptions = buildOptions(VertxCatsServerOptions.customiseInterceptors[IO](dispatcher), withServerLog)
     val interpreter = VertxCatsServerInterpreter(serverOptions)
     genEndpointsIO(nRoutes).map(interpreter.route(_)(router)).last
   }
 }
 
-class VertxCatsRunner(numRoutes: Int) {
+class VertxCatsRunner(numRoutes: Int, withServerLog: Boolean = false) {
 
   def start: IO[ServerRunner.KillSwitch] =
     Dispatcher.parallel[IO].allocated.flatMap { case (dispatcher, releaseDispatcher) =>
-      VertxRunner.runServer(Tapir.route(dispatcher)(numRoutes)).map(releaseVertx => releaseVertx >> releaseDispatcher)
+      VertxRunner.runServer(Tapir.route(dispatcher, withServerLog)(numRoutes)).map(releaseVertx => releaseVertx >> releaseDispatcher)
     }
 }
 
 object TapirServer extends VertxCatsRunner(numRoutes = 1) with ServerRunner
-object TapirMultiServer extends VertxCatsRunner(numRoutes = 1) with ServerRunner
+object TapirMultiServer extends VertxCatsRunner(numRoutes = 128) with ServerRunner
+object TapirInterceptorMultiServer extends VertxCatsRunner(numRoutes = 128, withServerLog = true) with ServerRunner

perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteParams.scala

@@ -14,9 +14,21 @@ case class PerfTestSuiteParams(
     durationSeconds: Int = PerfTestSuiteParams.defaultDurationSeconds,
     buildGatlingReports: Boolean = false
 ) {
+  /**
+    * Handles server names passed as groups like netty.*, pekko.*, etc. by expanding them into lists of actual server names.
+    * Similarly, handles '*' as a short simulation name, expanding it to a list of all simulations.
+    * @return
+    */
   def adjustWildcards: PerfTestSuiteParams = {
-    val withAdjustedServer: PerfTestSuiteParams =
-      if (shortServerNames == List("*")) copy(shortServerNames = TypeScanner.allServers) else this
+    val withAdjustedServer: PerfTestSuiteParams = {
+      val expandedShortServerNames = shortServerNames.flatMap { shortServerName =>
+        if (shortServerName.contains("*")) {
+          TypeScanner.allServers.filter(_.startsWith(shortServerName.stripSuffix("*")))
+        }
+        else List(shortServerName)
+      }
+      copy(shortServerNames = expandedShortServerNames)
+    }
     if (shortSimulationNames == List("*"))
       withAdjustedServer.copy(shortSimulationNames = TypeScanner.allSimulations)
     else
@@ -49,7 +61,7 @@ object PerfTestSuiteParams {
     opt[Seq[String]]('s', "server")
       .required()
       .action((x, c) => c.copy(shortServerNames = x.toList))
-      .text(s"Comma-separated list of short server names, or '*' for all. Available servers: ${TypeScanner.allServers.mkString(", ")}"),
+      .text(s"Comma-separated list of short server names, or groups like 'netty.*', 'pekko.*', etc. Available servers: ${TypeScanner.allServers.mkString(", ")}"),
     opt[Seq[String]]('m', "sim")
       .required()
       .action((x, c) => c.copy(shortSimulationNames = x.toList))