Parameterized gradle version feature

* Allow tests to be parameterized with Gradle versions.
* Document that code coverage does not work on Gradle 8.7 
* Fix temp directory cleanup

README.md

@@ -57,8 +57,34 @@ class MyTest {
 }
 ```
 
+## Parameterized Gradle versions
+
+To run a test against multiple versions of Gradle, use the `@ParameterizedWithGradleVersions` annotation.
+Gradle versions can be specified per class in the `@TestKit` annotation or per method in the 
+`@ParameterizedWithGradleVersions` annotation. Each gradle version argument will be automatically 
+injected into the runner created via `TestProject.createRunner`.
+
+```kotlin
+@TestKit(gradleVersions = ["8.6", "8.7"])
+class ParameterizedTest {
+    @Test
+    @ParameterizedWithGradleVersions
+    fun sometest(project: TestProject) {
+        project.createRunner()
+            .withArguments("check")
+            .build() 
+    }
+}
+```
+
 ## Code coverage
 
+> [!WARNING]  
+> Code coverage collection does not work on Gradle 8.7. 
+> You have to run tests against Gradle 8.6 or below to collect coverage.
+> You can use the parameterized Gradle version feature described above
+> to run tests against both older and newer versions of Gradle.
+
 It is notoriously difficult to collect code coverage data from TestKit tests. The root of the challenge
 is that by default, TestKit tests launch in a separate Gradle daemon JVM, which lingers after the tests finish. 
 This presents the following problems

coverage-plugin/src/test/kotlin/com/toasttab/gradle/testkit/FlushJacocoPluginIntegrationTest.kt

@@ -23,7 +23,6 @@ import org.junit.jupiter.api.io.TempDir
 import strikt.api.expectThat
 import strikt.assertions.contains
 import java.nio.file.Path
-import kotlin.io.path.Path
 import kotlin.io.path.inputStream
 import kotlin.io.path.writeText
 
@@ -50,6 +49,7 @@ class FlushJacocoPluginIntegrationTest {
         )
 
         GradleRunner.create()
+            .withGradleVersion("8.6")
             .withProjectDir(dir.toFile())
             .withPluginClasspath().withArguments("build").build()
 

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

junit5/src/main/kotlin/com/toasttab/gradle/testkit/CoverageRecorder.kt

@@ -28,11 +28,11 @@ import java.net.ServerSocket
 import kotlin.concurrent.thread
 
 internal class CoverageRecorder(
-    outputFile: String
+    settings: CoverageSettings
 ) : ExtensionContext.Store.CloseableResource, ISessionInfoVisitor, IExecutionDataVisitor {
     private val server = ServerSocket(0)
 
-    private val output = FileOutputStream(outputFile)
+    private val output = FileOutputStream(settings.output, true)
     private val writer = ExecutionDataWriter(output)
 
     private val runner = thread {

junit5/src/main/kotlin/com/toasttab/gradle/testkit/GradleVersionArgument.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2024 Toast Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.toasttab.gradle.testkit
+
+class GradleVersionArgument private constructor(
+    val version: String?
+) {
+    companion object {
+        fun of(version: String) = GradleVersionArgument(version)
+
+        val DEFAULT = GradleVersionArgument(null)
+    }
+}

junit5/src/main/kotlin/com/toasttab/gradle/testkit/ParameterizedWithGradleVersions.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2024 Toast Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.toasttab.gradle.testkit
+
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ArgumentsSource
+
+@ArgumentsSource(TestProjectExtension::class)
+@ParameterizedTest
+annotation class ParameterizedWithGradleVersions(val value: Array<String> = [])

junit5/src/main/kotlin/com/toasttab/gradle/testkit/TestKit.kt

@@ -23,5 +23,6 @@ import kotlin.reflect.KClass
 @ExtendWith(TestProjectExtension::class)
 annotation class TestKit(
     val locator: KClass<out ProjectLocator> = SimpleNameProjectLocator::class,
+    val gradleVersions: Array<String> = [],
     val cleanup: Boolean = true
 )

junit5/src/main/kotlin/com/toasttab/gradle/testkit/TestProject.kt

@@ -17,24 +17,27 @@ package com.toasttab.gradle.testkit
 
 import org.gradle.testkit.runner.GradleRunner
 import org.slf4j.LoggerFactory
-import java.io.Closeable
 import java.io.StringWriter
 import java.nio.file.Path
+import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.io.path.ExperimentalPathApi
 import kotlin.io.path.deleteRecursively
 
 class TestProject(
     val dir: Path,
-    private val cleanup: Boolean
-) : Closeable {
+    private val gradleVersion: GradleVersionArgument,
+    private val cleanup: Boolean,
+) {
+
     companion object {
         private val LOGGER = LoggerFactory.getLogger(TestProject::class.java)
     }
 
     private val output = StringWriter()
+    private val outputLogged = AtomicBoolean()
 
     @OptIn(ExperimentalPathApi::class)
-    override fun close() {
+    fun close() {
         if (cleanup) {
             dir.deleteRecursively()
         }
@@ -46,9 +49,15 @@ class TestProject(
     fun createRunnerWithoutPluginClasspath() = GradleRunner.create()
         .withProjectDir(dir.toFile())
         .forwardStdOutput(output)
-        .forwardStdError(output)
+        .forwardStdError(output).apply {
+            if (gradleVersion.version != null) {
+                withGradleVersion(gradleVersion.version)
+            }
+        }
 
-    fun logOutput() {
-        LOGGER.warn("build output:\n{}", output)
+    fun logOutputOnce() {
+        if (!outputLogged.getAndSet(true)) {
+            LOGGER.warn("build output:\n{}", output)
+        }
     }
 }

junit5/src/main/kotlin/com/toasttab/gradle/testkit/TestProjectExtension.kt

@@ -19,80 +19,126 @@ import org.gradle.testkit.runner.UnexpectedBuildResultException
 import org.junit.jupiter.api.extension.AfterTestExecutionCallback
 import org.junit.jupiter.api.extension.BeforeAllCallback
 import org.junit.jupiter.api.extension.ExtensionContext
+import org.junit.jupiter.api.extension.ExtensionContext.Store.CloseableResource
+import org.junit.jupiter.api.extension.InvocationInterceptor
 import org.junit.jupiter.api.extension.ParameterContext
 import org.junit.jupiter.api.extension.ParameterResolver
+import org.junit.jupiter.params.provider.Arguments
+import org.junit.jupiter.params.provider.ArgumentsProvider
+import java.util.*
+import java.util.concurrent.ConcurrentHashMap
+import java.util.stream.Stream
 import kotlin.io.path.ExperimentalPathApi
 import kotlin.io.path.appendText
 import kotlin.io.path.copyToRecursively
 import kotlin.io.path.createFile
 import kotlin.io.path.createTempDirectory
 import kotlin.io.path.exists
 
-private val NAMESPACE = ExtensionContext.Namespace.create("testkit-project")
+private val NAMESPACE = ExtensionContext.Namespace.create(TestProjectExtension::class.java.name, "testkit-project")
 private const val COVERAGE_RECORDER = "coverage-recorder"
-private const val PROJECT = "project"
 private const val LOCATOR = "locator"
+private const val PROJECTS = "PROJECTS"
 
-class TestProjectExtension : ParameterResolver, BeforeAllCallback, AfterTestExecutionCallback {
+data class ProjectKey(
+    val gradleVersion: String?
+) {
+    constructor(gradleVersion: GradleVersionArgument) : this(gradleVersion.version)
+}
+
+class TestProjects : CloseableResource {
+    private val projects = ConcurrentHashMap<ProjectKey, TestProject>()
+
+    fun project(key: ProjectKey, create: (ProjectKey) -> TestProject) = projects.computeIfAbsent(key, create)
+
+    override fun close() {
+        projects.values.forEach(TestProject::close)
+    }
+
+    fun logOutputOnce() {
+        projects.values.forEach(TestProject::logOutputOnce)
+    }
+}
+
+class TestProjectExtension : ParameterResolver, BeforeAllCallback, AfterTestExecutionCallback, InvocationInterceptor, ArgumentsProvider {
     override fun beforeAll(context: ExtensionContext) {
         val coverage = CoverageSettings.settings
 
         if (coverage != null) {
-            context.getStore(NAMESPACE).put(COVERAGE_RECORDER, CoverageRecorder(coverage.output))
+            context.getStore(NAMESPACE).put(COVERAGE_RECORDER, CoverageRecorder(coverage))
+        }
+    }
+
+    override fun supportsParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext) =
+        parameterContext.parameter.type == TestProject::class.java &&
+            extensionContext.parent.map { it::class.java.name } != Optional.of("org.junit.jupiter.engine.descriptor.TestTemplateExtensionContext")
+
+    override fun resolveParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext) =
+        extensionContext.project(GradleVersionArgument.DEFAULT)
+
+    override fun afterTestExecution(context: ExtensionContext) {
+        context.executionException.ifPresent {
+            if (it !is UnexpectedBuildResultException) {
+                context.get<TestProjects>(NAMESPACE, PROJECTS).logOutputOnce()
+            }
         }
     }
 
-    @OptIn(ExperimentalPathApi::class)
-    private fun project(context: ExtensionContext): TestProject {
-        val parameters = context.requiredTestClass.getAnnotation(TestKit::class.java) ?: TestKit()
+    override fun provideArguments(context: ExtensionContext): Stream<out Arguments> {
+        val methodAnn = context.requiredTestMethod.getAnnotation(ParameterizedWithGradleVersions::class.java)
 
-        val locator = context.getStore(NAMESPACE).cache(LOCATOR) {
-            parameters.locator.java.getDeclaredConstructor().newInstance() as ProjectLocator
+        val versions = if (methodAnn.value.isNotEmpty()) {
+            methodAnn.value
+        } else {
+            context.requiredTestClass.getAnnotation(TestKit::class.java).gradleVersions
         }
 
-        return context.getStore(NAMESPACE).cache(PROJECT) {
-            val tempProjectDir = createTempDirectory("junit-gradlekit")
+        return versions.map {
+            Arguments.of(context.project(GradleVersionArgument.of(it)))
+        }.stream()
+    }
+}
 
-            val location = locator.projectPath(System.getProperty(TESTKIT_PROJECTS), context)
+private inline fun <reified T> ExtensionContext.get(namespace: ExtensionContext.Namespace, key: String) =
+    getStore(namespace).get(key, T::class.java)
 
-            if (!location.exists()) {
-                error { "expected a test project in $location" }
-            }
+private inline fun <K, reified V> ExtensionContext.Store.cache(key: K, noinline f: (K) -> V) =
+    getOrComputeIfAbsent(key, f, V::class.java)
 
-            location.copyToRecursively(target = tempProjectDir, followLinks = false, overwrite = false)
+@OptIn(ExperimentalPathApi::class)
+private fun ExtensionContext.project(gradleVersion: GradleVersionArgument): TestProject {
+    val parameters = requiredTestClass.getAnnotation(TestKit::class.java) ?: TestKit()
 
-            val coverage = CoverageSettings.settings
+    val locator = getStore(NAMESPACE).cache(LOCATOR) {
+        parameters.locator.java.getDeclaredConstructor().newInstance() as ProjectLocator
+    }
 
-            if (coverage != null) {
-                val collector = context.get<CoverageRecorder>(NAMESPACE, COVERAGE_RECORDER)
-                tempProjectDir.resolve("gradle.properties").apply {
-                    if (!exists()) {
-                        createFile()
-                    }
+    return getStore(NAMESPACE).cache(PROJECTS) {
+        TestProjects()
+    }.project(ProjectKey(gradleVersion)) {
+        val tempProjectDir = createTempDirectory("junit-gradlekit")
 
-                    appendText("\norg.gradle.jvmargs=-javaagent:${coverage.javaagent}=output=tcpclient,port=${collector.port},sessionid=test,includes=${coverage.includes},excludes=${coverage.excludes}\n")
-                }
-            }
+        val location = locator.projectPath(System.getProperty(TESTKIT_PROJECTS), this)
 
-            TestProject(tempProjectDir, parameters.cleanup)
+        if (!location.exists()) {
+            error { "expected a test project in $location" }
         }
-    }
 
-    private inline fun <reified T> ExtensionContext.get(namespace: ExtensionContext.Namespace, key: String) = getStore(namespace).get(key, T::class.java)
-    private inline fun <K, reified V> ExtensionContext.Store.cache(key: K, noinline f: (K) -> V) =
-        getOrComputeIfAbsent(key, f, V::class.java)
+        location.copyToRecursively(target = tempProjectDir, followLinks = false, overwrite = false)
 
-    override fun supportsParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext) =
-        parameterContext.parameter.type == TestProject::class.java
+        val coverage = CoverageSettings.settings
 
-    override fun resolveParameter(parameterContext: ParameterContext, extensionContext: ExtensionContext) =
-        project(extensionContext)
+        if (coverage != null) {
+            val collector = get<CoverageRecorder>(NAMESPACE, COVERAGE_RECORDER)
+            tempProjectDir.resolve("gradle.properties").apply {
+                if (!exists()) {
+                    createFile()
+                }
 
-    override fun afterTestExecution(context: ExtensionContext) {
-        context.executionException.ifPresent {
-            if (it !is UnexpectedBuildResultException) {
-                context.get<TestProject>(NAMESPACE, PROJECT).logOutput()
+                appendText("\norg.gradle.jvmargs=-javaagent:${coverage.javaagent}=output=tcpclient,port=${collector.port},sessionid=test,includes=${coverage.includes},excludes=${coverage.excludes}\n")
             }
         }
+
+        TestProject(tempProjectDir, gradleVersion, parameters.cleanup)
     }
 }

junit5/src/test/kotlin/com/toasttab/gradle/testkit/TestKitIntegrationTest.kt

@@ -19,12 +19,21 @@ import org.junit.jupiter.api.Test
 import strikt.api.expectThat
 import strikt.assertions.contains
 
-@TestKit
+@TestKit(gradleVersions = ["8.6", "8.7"])
 class TestKitIntegrationTest {
     @Test
     fun `basic project`(project: TestProject) {
         expectThat(
             project.createRunnerWithoutPluginClasspath().withArguments("dependencies").build().output
         ).contains("compileClasspath")
     }
+
+    @ParameterizedWithGradleVersions
+    fun `basic parameterized project`(project: TestProject) {
+        val output = project.createRunnerWithoutPluginClasspath().withArguments("dependencies").build().output
+
+        expectThat(
+            output
+        ).contains("compileClasspath")
+    }
 }

junit5/src/test/projects/TestKitIntegrationTest/basic parameterized project/build.gradle.kts

@@ -0,0 +1,5 @@
+plugins {
+    java
+}
+
+println("gradle version = ${gradle.gradleVersion}")

junit5/src/test/projects/TestKitIntegrationTest/basic parameterized project/settings.gradle.kts

@@ -0,0 +1 @@
+rootProject.name = "test"