Install plugins into an integration repo

README.md

@@ -35,11 +35,19 @@ src/test/projects/MyTest/sometest:
    settings.gradle.kts # optional, but IntelliJ will complain
 ```
 
-In the test project's `build.gradle.kts`, make sure to apply the coverage plugin.
+Test project files may contain Ant-style placeholders. The predefined placeholders are:
 
-```
+* `@TESTKIT_PLUGIN_VERSION@` - the version of this project
+* `@TESTKIT_INTEGRATION_REPO@` - the location of the integration repository, see below
+* `@VERSION@` - the version of the plugin under test
+
+In the test project's `build.gradle.kts`, make sure to apply the coverage plugin, in addition to the plugin under test.
+Note that both plugins require versions which can be specified using the placeholders above.
+
+```kotlin
 plugins {
-    id("com.toasttab.testkit.coverage")
+    id("com.toasttab.testkit.coverage") version "@TESTKIT_PLUGIN_VERSION@"
+    id("my.plugin.under.test") version "@VERSION@"
 }
 ```
 
@@ -50,9 +58,7 @@ Now, write the actual test. Note that a `TestProject` instance will be automatic
 class MyTest {
     @Test
     fun sometest(project: TestProject) {
-        project.createRunner()
-            .withArguments("check")
-            .build()
+        project.build("check")
     }
 }
 ```
@@ -77,6 +83,16 @@ class ParameterizedTest {
 }
 ```
 
+## Integration repository
+
+This plugin does not use the TestKit's plugin classpath injection mechanism because the mechanism breaks
+in certain scenarios, e.g. when plugins depend on other plugins. Instead, this plugin installs
+the plugin under test and its sibling dependencies, optionally preinstrumented for Jacoco code coverage,
+into an integration repository on disk, an technique borrowed from [DAGP](https://github.com/autonomousapps/dependency-analysis-gradle-plugin). 
+
+The integration repository is then injected into the plugin management repositories via a custom init
+script which is generated on the fly.
+
 ## Code coverage
 
 It is notoriously difficult to collect code coverage data from TestKit tests. By default, TestKit tests launch in 

coverage-plugin/build.gradle.kts

@@ -1,12 +1,30 @@
 plugins {
     `kotlin-conventions`
-    `library-publishing-conventions`
+    `plugin-publishing-conventions`
     jacoco
 }
 
+gradlePlugin {
+    plugins {
+        create("jacoco") {
+            id = "com.toasttab.testkit.coverage"
+            implementationClass = "com.toasttab.gradle.testkit.FlushJacocoPlugin"
+            description = ProjectInfo.description
+            displayName = ProjectInfo.name
+            tags = listOf("jacoco", "testkit")
+        }
+    }
+}
+
 dependencies {
     implementation(gradleApi())
     implementation(projects.jacocoReflect)
+    implementation(libs.jacoco.agent) {
+        artifact {
+            classifier = "runtime"
+            extension = "jar"
+        }
+    }
 
     testImplementation(projects.junit5)
     testImplementation(libs.junit)

coverage-plugin/src/main/kotlin/com/toasttab/gradle/testkit/FlushJacocoPlugin.kt

@@ -33,6 +33,9 @@ class FlushJacocoPlugin @Inject constructor(
 
 class DumpAction : FlowAction<FlowParameters.None> {
     override fun execute(parameters: FlowParameters.None) {
-        JacocoRt.requiredAgent.dump(false)
+        JacocoRt.requiredAgent.run {
+            writeExecutionData(true)
+            shutdown()
+        }
     }
 }

gradle/libs.versions.toml

@@ -19,6 +19,7 @@ spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.re
 gradle-publish = { module = "com.gradle.publish:plugin-publish-plugin", version = "1.2.0" }
 
 jacoco-core = { module = "org.jacoco:org.jacoco.core", version.ref = "jacoco" }
+jacoco-agent = { module = "org.jacoco:org.jacoco.agent", version.ref = "jacoco" }
 
 # test
 junit = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }

integration-tests/build.gradle.kts

@@ -1,11 +1,34 @@
-import com.toasttab.gradle.testkit.shared.configureInstrumentation
+import com.toasttab.gradle.testkit.shared.RepositoryDescriptor
+import com.toasttab.gradle.testkit.shared.configureIntegrationPublishing
+import com.toasttab.gradle.testkit.shared.publishOnlyIf
 
 plugins {
     `kotlin-conventions`
     jacoco
+    `plugin-publishing-conventions`
 }
 
-configureInstrumentation(projects.coveragePlugin)
+gradlePlugin {
+    plugins {
+        create("integration") {
+            id = "com.toasttab.testkit.integration.test"
+            implementationClass = "com.toasttab.gradle.testkit.TestPlugin"
+            description = ProjectInfo.description
+            displayName = ProjectInfo.name
+            tags = listOf("jacoco", "testkit")
+        }
+    }
+}
+
+tasks {
+    test {
+        systemProperty("version", "$version")
+        systemProperty("testkit-integration-repo", rootProject.layout.buildDirectory.dir("integration-repo").get().asFile.path)
+    }
+}
+
+configureIntegrationPublishing("testRuntimeClasspath")
+publishOnlyIf { _, repo -> repo == RepositoryDescriptor.INTEGRATION }
 
 dependencies {
     implementation(gradleApi())
@@ -15,4 +38,5 @@ dependencies {
     testImplementation(projects.junit5)
     testImplementation(gradleTestKit())
     testImplementation(libs.jacoco.core)
+    testImplementation(projects.coveragePlugin)
 }

integration-tests/src/test/kotlin/com/toasttab/gradle/testkit/FlushJacocoPluginIntegrationTest.kt

@@ -15,7 +15,6 @@
 
 package com.toasttab.gradle.testkit
 
-import org.gradle.testkit.runner.GradleRunner
 import org.jacoco.core.data.ExecutionDataReader
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
@@ -31,6 +30,8 @@ class FlushJacocoPluginIntegrationTest {
 
     @Test
     fun `coverage is flushed`() {
+        val version = System.getProperty("version")
+
         val file = dir.resolve("build/testkit.exec")
 
         dir.resolve("gradle.properties").writeText(
@@ -41,18 +42,13 @@ class FlushJacocoPluginIntegrationTest {
             """
                 plugins {
                     java
-                    id("com.toasttab.testkit.coverage")
-                    id("com.toasttab.testkit.test")
+                    id("com.toasttab.testkit.coverage") version("$version")
+                    id("com.toasttab.testkit.integration.test") version("$version")
                 }
             """.trimIndent()
         )
 
-        GradleRunner.create()
-            .withGradleVersion("8.7")
-            .withProjectDir(dir.toFile())
-            .let(TestProjectExtension.pluginClasspath()::apply)
-            .withArguments("build", "--configuration-cache")
-            .build()
+        TestProjectExtension.createProject(dir, GradleVersionArgument.of("8.7")).build("build", "--configuration-cache", "--stacktrace")
 
         val classes = hashSetOf<String>()
 

jacoco-reflect/src/main/kotlin/com/toasttab/gradle/testkit/jacoco/JacocoRt.kt

@@ -17,11 +17,14 @@ package com.toasttab.gradle.testkit.jacoco
 
 interface JacocoAgent {
     val location: String
-    fun dump(reset: Boolean)
+
+    fun writeExecutionData(reset: Boolean)
 
     val includes: String
 
     val excludes: String
+
+    fun shutdown()
 }
 
 private class ReflectiveJacocoAgent(
@@ -32,15 +35,24 @@ private class ReflectiveJacocoAgent(
         agent.javaClass.getDeclaredField("options").apply { isAccessible = true }.get(agent)
     }
 
+    private val output by lazy {
+        // agent.output
+        agent.javaClass.getDeclaredField("output").apply { isAccessible = true }.get(agent)
+    }
+
     override val location: String
         get() = agent.javaClass.protectionDomain.codeSource.location.file
     override val includes: String
         get() = options.javaClass.getMethod("getIncludes").invoke(options) as String
     override val excludes: String
         get() = options.javaClass.getMethod("getExcludes").invoke(options) as String
 
-    override fun dump(reset: Boolean) {
-        agent.javaClass.getMethod("dump", Boolean::class.java).invoke(agent, reset)
+    override fun writeExecutionData(reset: Boolean) {
+        output.javaClass.getMethod("writeExecutionData", Boolean::class.java).invoke(output, reset)
+    }
+
+    override fun shutdown() {
+        output.javaClass.getMethod("shutdown").invoke(output)
     }
 }
 

junit5/src/main/kotlin/com/toasttab/gradle/testkit/CoverageRecorder.kt

@@ -15,11 +15,7 @@
 
 package com.toasttab.gradle.testkit
 
-import org.jacoco.core.data.ExecutionData
 import org.jacoco.core.data.ExecutionDataWriter
-import org.jacoco.core.data.IExecutionDataVisitor
-import org.jacoco.core.data.ISessionInfoVisitor
-import org.jacoco.core.data.SessionInfo
 import org.jacoco.core.runtime.RemoteControlReader
 import org.jacoco.core.runtime.RemoteControlWriter
 import org.junit.jupiter.api.extension.ExtensionContext
@@ -29,42 +25,53 @@ import kotlin.concurrent.thread
 
 internal class CoverageRecorder(
     settings: CoverageSettings
-) : ExtensionContext.Store.CloseableResource, ISessionInfoVisitor, IExecutionDataVisitor {
+) : ExtensionContext.Store.CloseableResource {
     private val server = ServerSocket(0)
 
     private val output = FileOutputStream(settings.output, true)
     private val writer = ExecutionDataWriter(output)
 
+    private val threads = mutableListOf<Thread>()
+
     private val runner = thread {
-        val sock = server.accept()
+        while (!server.isClosed) {
+            val sock = server.accept()
 
-        RemoteControlWriter(sock.getOutputStream())
+            threads.add(
+                thread {
+                    RemoteControlWriter(sock.getOutputStream())
 
-        val reader = RemoteControlReader(sock.getInputStream())
-        reader.setSessionInfoVisitor(this)
-        reader.setExecutionDataVisitor(this)
+                    val reader = RemoteControlReader(sock.getInputStream())
+                    reader.setSessionInfoVisitor { }
 
-        while (reader.read()) {
-        }
+                    reader.setExecutionDataVisitor { ex ->
+                        synchronized(writer) {
+                            writer.visitClassExecution(ex)
+                        }
+                    }
 
-        writer.flush()
-        sock.close()
-    }
+                    reader.setRemoteCommandVisitor { _, _ -> }
 
-    val port: Int get() = server.localPort
+                    while (reader.read()) {
+                    }
 
-    override fun visitSessionInfo(sess: SessionInfo) {
-        writer.visitSessionInfo(sess)
+                    synchronized(writer) {
+                        writer.flush()
+                    }
+                    sock.close()
+                }
+            )
+        }
     }
 
-    override fun visitClassExecution(ex: ExecutionData) {
-        writer.visitClassExecution(ex)
-    }
+    val port: Int get() = server.localPort
 
     override fun close() {
-        runner.join(10000)
-        server.close()
+        for (thread in threads) {
+            thread.join(10000)
+        }
 
+        server.close()
         output.close()
     }
 }

junit5/src/main/kotlin/com/toasttab/gradle/testkit/TestProject.kt

@@ -23,27 +23,12 @@ import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.io.path.ExperimentalPathApi
 import kotlin.io.path.deleteRecursively
 
-sealed interface PluginClasspath {
-    fun apply(runner: GradleRunner): GradleRunner
-
-    object Default : PluginClasspath {
-        override fun apply(runner: GradleRunner) = runner.withPluginClasspath()
-    }
-
-    class Custom(
-        private val paths: List<Path>
-    ) : PluginClasspath {
-        override fun apply(runner: GradleRunner) = runner.withPluginClasspath(paths.map(Path::toFile))
-    }
-}
-
 class TestProject(
     val dir: Path,
-    private val classpath: PluginClasspath,
     private val gradleVersion: GradleVersionArgument,
     private val cleanup: Boolean,
+    private val initArgs: List<String> = emptyList()
 ) {
-
     companion object {
         private val LOGGER = LoggerFactory.getLogger(TestProject::class.java)
     }
@@ -58,16 +43,21 @@ class TestProject(
         }
     }
 
-    fun createRunner() = createRunnerWithoutPluginClasspath().let(classpath::apply)
+    fun build(vararg args: String) = createRunner(*args).build()
+
+    fun buildAndFail(vararg args: String) = createRunner(*args).buildAndFail()
+
+    private fun createRunner(vararg args: String) = createRunner().withArguments(initArgs + args)
 
-    fun createRunnerWithoutPluginClasspath() = GradleRunner.create()
+    private fun createRunner() = GradleRunner.create()
         .withProjectDir(dir.toFile())
         .forwardStdOutput(output)
         .forwardStdError(output).apply {
             if (gradleVersion.version != null) {
                 withGradleVersion(gradleVersion.version)
             }
         }
+        .withArguments()
 
     fun logOutputOnce() {
         if (!outputLogged.getAndSet(true)) {