Merge pull request #102 from bjaglin/caching

Opt-in caching for scalafix invocations

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
       matrix:
         command:
           - "++2.12.11 test scripted"
-          - "++2.10.7 test 'scripted sbt-scalafix/*'"
+          - "++2.10.7 test 'scripted sbt-scalafix/* skip-windows/*'"
     steps:
       - uses: actions/checkout@v2
       - uses: olafurpg/setup-scala@v7

build.sbt

@@ -27,7 +27,7 @@ developers := List(
 
 commands += Command.command("ci-windows") { s =>
   "testOnly -- -l SkipWindows" ::
-    "scripted" ::
+    "scripted sbt-*/*" ::
     s
 }
 

src/main/scala-sbt-0.13/sbt/internal/sbtscalafix/Caching.scala

@@ -0,0 +1,44 @@
+package sbt.internal.sbtscalafix
+
+import java.io.ByteArrayOutputStream
+
+import sbinary._
+import sbt._
+import scalafix.internal.sbt.Arg
+
+import scala.util.DynamicVariable
+
+object Caching {
+
+  val lastModifiedStyle = FilesInfo.lastModified
+
+  trait CacheKeysStamper
+      extends InputCache[Seq[Arg.CacheKey]]
+      with CacheImplicits {
+    private val output = new DynamicVariable[Output](null)
+
+    implicit val lastModifiedFormat = FileInfo.lastModified.format
+    val baFormat = implicitly[Format[Array[Byte]]]
+    val baEquiv = implicitly[Equiv[Array[Byte]]]
+
+    protected def stamp: Arg.CacheKey => Unit
+
+    final def write[T](t: T)(implicit format: Format[T]): Unit =
+      format.writes(output.value, t)
+
+    override final type Internal = Array[Byte]
+    override final def convert(keys: Seq[Arg.CacheKey]): Array[Byte] = {
+      val baos = new ByteArrayOutputStream()
+      output.withValue(new JavaOutput(baos)) {
+        keys.foreach(stamp)
+      }
+      Hash.apply(baos.toByteArray)
+    }
+    override final def read(from: Input): Array[Byte] =
+      baFormat.reads(from)
+    override final def write(to: Output, ba: Array[Byte]): Unit =
+      baFormat.writes(to, ba)
+    override final def equiv: Equiv[Array[Byte]] =
+      baEquiv
+  }
+}

src/main/scala-sbt-1.0/sbt/internal/sbtscalafix/Caching.scala

@@ -0,0 +1,38 @@
+package sbt.internal.sbtscalafix
+
+import sbt.FileInfo
+import scalafix.internal.sbt.Arg
+import sjsonnew._
+
+import scala.util.DynamicVariable
+
+object Caching {
+
+  val lastModifiedStyle = FileInfo.lastModified
+
+  trait CacheKeysStamper
+      extends JsonFormat[Seq[Arg.CacheKey]]
+      with BasicJsonProtocol {
+    private val builder = new DynamicVariable[Builder[_]](null)
+
+    protected def stamp: Arg.CacheKey => Unit
+
+    final def write[T](t: T)(implicit format: JsonFormat[T]): Unit =
+      format.write(t, builder.value)
+
+    override final def write[J](obj: Seq[Arg.CacheKey], b: Builder[J]): Unit = {
+      b.beginArray()
+      builder.withValue(b) {
+        obj.foreach(stamp)
+      }
+      b.endArray()
+    }
+
+    // we actually don't need to read anything back, see https://github.com/sbt/sbt/pull/5513
+    override final def read[J](
+        jsOpt: Option[J],
+        unbuilder: Unbuilder[J]
+    ): Seq[Arg.CacheKey] = ???
+  }
+
+}

src/main/scala/scalafix/internal/sbt/ScalafixCompletions.scala

@@ -145,10 +145,13 @@ class ScalafixCompletions(
         "--syntactic",
         "--verbose",
         "--version"
-      ).map(literal)
+      ).map(f => literal(f).map(ShellArgs.Extra(_))) ++ Seq(
+        "--no-cache".^^^(ShellArgs.NoCache)
+      )
       Parser.oneOf(flags) |
         hide(string) // catch-all for all args not known to sbt-scalafix
-    }.map(ShellArgs.Extra(_))
+          .map(ShellArgs.Extra(_))
+    }
 
     val rule: ArgP =
       fileRule |

src/main/scala/scalafix/internal/sbt/ScalafixInterface.scala

@@ -1,25 +1,111 @@
 package scalafix.internal.sbt
 
+import java.io.PrintStream
 import java.net.URLClassLoader
+import java.nio.file.Path
+import java.{util => jutil}
 
 import com.geirsson.coursiersmall.Repository
 import sbt._
 import sbt.internal.sbtscalafix.Compat
-import scalafix.interfaces.{ScalafixArguments, Scalafix => ScalafixAPI}
+import scalafix.interfaces.{Scalafix => ScalafixAPI, _}
+import scalafix.sbt.InvalidArgument
+
+import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
+
+sealed trait Arg extends (ScalafixArguments => ScalafixArguments)
+
+object Arg {
+
+  sealed trait CacheKey
+
+  case class ToolClasspath(classLoader: URLClassLoader)
+      extends Arg
+      with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      // this effectively overrides any previous URLClassLoader
+      sa.withToolClasspath(classLoader)
+  }
+
+  case class Rules(rules: Seq[String]) extends Arg with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa.withRules(rules.asJava)
+  }
+
+  case class Paths(paths: Seq[Path]) extends Arg { // this one is extracted/stamped directly
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa.withPaths(paths.asJava)
+  }
+
+  case class Config(file: Option[Path]) extends Arg with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa.withConfig(jutil.Optional.ofNullable(file.orNull))
+  }
+
+  case class ParsedArgs(args: Seq[String]) extends Arg with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa.withParsedArguments(args.asJava)
+  }
+
+  case class ScalaVersion(version: String) extends Arg { //FIXME: with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa.withScalaVersion(version)
+  }
+
+  case class ScalacOptions(options: Seq[String]) extends Arg { //FIXME: with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa.withScalacOptions(options.asJava)
+  }
+
+  case class Classpath(classpath: Seq[Path]) extends Arg { //FIXME: with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa.withClasspath(classpath.asJava)
+  }
+
+  case object NoCache extends Arg with CacheKey {
+    override def apply(sa: ScalafixArguments): ScalafixArguments =
+      sa // caching is currently implemented in sbt-scalafix itself
+  }
+}
 
 class ScalafixInterface private (
-    val args: ScalafixArguments,
-    private val toolClasspath: URLClassLoader
+    scalafixArguments: ScalafixArguments, // hide it to force usage of withArgs so we can intercept arguments
+    val args: Seq[Arg]
 ) {
 
-  // Accumulates the classpath via classloader delegation, as args.withToolClasspath() only considers the last call.
+  private val lastToolClasspath = args.reverse
+    .collectFirst { case tcp: Arg.ToolClasspath => tcp }
+    .getOrElse(
+      throw new IllegalArgumentException(
+        "a base toolClasspath must be provided"
+      )
+    )
+    .classLoader
+
+  private def this(
+      api: ScalafixAPI,
+      toolClasspath: URLClassLoader,
+      mainCallback: ScalafixMainCallback,
+      printStream: PrintStream
+  ) = this(
+    api
+      .newArguments()
+      .withMainCallback(mainCallback)
+      .withPrintStream(printStream)
+      .withToolClasspath(toolClasspath),
+    Seq(Arg.ToolClasspath(toolClasspath))
+  )
+
+  // Accumulates the classpath via classloader delegation, as only the last Arg.ToolClasspath is considered
   //
   // We effectively end up with the following class loader hierarchy:
   //   1. Meta-project sbt class loader
   //       - bound to the sbt session
   //   2. ScalafixInterfacesClassloader, loading `scalafix-interfaces` from its parent
   //       - bound to the sbt session
   //   3. `scalafix-cli` JARs
+  //       - passed in the constructor
   //       - bound to the sbt session
   //   4. Global, external dependencies
   //       - present only if custom dependencies were defined
@@ -34,23 +120,40 @@ class ScalafixInterface private (
       customResolvers: Seq[Repository],
       extraInternalDeps: Seq[File]
   ): ScalafixInterface = {
-    if (extraExternalDeps.isEmpty && extraInternalDeps.isEmpty) {
-      this
-    } else {
-      val extraURLs = ScalafixCoursier
-        .scalafixToolClasspath(
-          extraExternalDeps,
-          customResolvers
-        ) ++ extraInternalDeps.map(_.toURI.toURL)
-
-      val newToolClasspath =
-        new URLClassLoader(extraURLs.toArray, toolClasspath)
-      new ScalafixInterface(
-        args.withToolClasspath(newToolClasspath),
-        newToolClasspath
-      )
+    val extraURLs = ScalafixCoursier
+      .scalafixToolClasspath(
+        extraExternalDeps,
+        customResolvers
+      ) ++ extraInternalDeps.map(_.toURI.toURL)
+
+    if (extraURLs.isEmpty) this
+    else {
+      val classpath = new URLClassLoader(extraURLs.toArray, lastToolClasspath)
+      withArgs(Arg.ToolClasspath(classpath))
     }
   }
+
+  def withArgs(args: Arg*): ScalafixInterface = {
+    val newScalafixArguments = args.foldLeft(scalafixArguments) { (acc, arg) =>
+      try arg(acc)
+      catch { case NonFatal(e) => throw new InvalidArgument(e.getMessage) }
+    }
+    new ScalafixInterface(newScalafixArguments, this.args ++ args)
+  }
+
+  def run(): Seq[ScalafixError] =
+    scalafixArguments.run().toSeq
+
+  def availableRules(): Seq[ScalafixRule] =
+    scalafixArguments.availableRules().asScala
+
+  def rulesThatWillRun(): Seq[ScalafixRule] =
+    try scalafixArguments.rulesThatWillRun().asScala
+    catch { case NonFatal(e) => throw new InvalidArgument(e.getMessage) }
+
+  def validate(): Option[ScalafixException] =
+    Option(scalafixArguments.validate().orElse(null))
+
 }
 
 object ScalafixInterface {
@@ -61,7 +164,8 @@ object ScalafixInterface {
   def fromToolClasspath(
       scalafixDependencies: Seq[ModuleID],
       scalafixCustomResolvers: Seq[Repository],
-      logger: Logger = Compat.ConsoleLogger(System.out)
+      logger: Logger = Compat.ConsoleLogger(System.out),
+      printStream: PrintStream = System.out
   ): () => ScalafixInterface =
     new LazyValue({ () =>
       val jars = ScalafixCoursier.scalafixCliJars(scalafixCustomResolvers)
@@ -71,15 +175,11 @@ object ScalafixInterface {
       val classloader = new URLClassLoader(urls, interfacesParent)
       val api = ScalafixAPI.classloadInstance(classloader)
       val callback = new ScalafixLogger(logger)
-
-      val args = api
-        .newArguments()
-        .withMainCallback(callback)
-
-      new ScalafixInterface(args, classloader).addToolClasspath(
-        scalafixDependencies,
-        scalafixCustomResolvers,
-        Nil
-      )
+      new ScalafixInterface(api, classloader, callback, printStream)
+        .addToolClasspath(
+          scalafixDependencies,
+          scalafixCustomResolvers,
+          Nil
+        )
     })
 }

src/main/scala/scalafix/internal/sbt/SemanticRuleValidator.scala

@@ -3,15 +3,14 @@ package scalafix.internal.sbt
 import java.nio.file.Path
 
 import sbt.{CrossVersion, ModuleID}
-import scalafix.interfaces.ScalafixArguments
 
 import scala.collection.mutable.ListBuffer
 
 class SemanticRuleValidator(ifNotFound: SemanticdbNotFound) {
   def findErrors(
       files: Seq[Path],
       dependencies: Seq[ModuleID],
-      args: ScalafixArguments
+      interface: ScalafixInterface
   ): Seq[String] = {
     if (files.isEmpty) Nil
     else {
@@ -20,9 +19,10 @@ class SemanticRuleValidator(ifNotFound: SemanticdbNotFound) {
         dependencies.exists(_.name.startsWith("semanticdb-scalac"))
       if (!hasSemanticdb)
         errors += ifNotFound.message
-      val invalidArguments = args.validate()
-      if (invalidArguments.isPresent)
-        errors += invalidArguments.get().getMessage
+      val invalidArguments = interface.validate()
+      invalidArguments.foreach { invalidArgument =>
+        errors += invalidArgument.getMessage
+      }
       errors
     }
   }

src/main/scala/scalafix/internal/sbt/ShellArgs.scala

@@ -6,22 +6,26 @@ object ShellArgs {
   case class Rule(value: String) extends Arg
   case class File(value: String) extends Arg
   case class Extra(key: String, value: Option[String] = None) extends Arg
+  case object NoCache extends Arg
 
   def apply(args: Seq[Arg]): ShellArgs = {
     val rules = List.newBuilder[String]
     val files = List.newBuilder[String]
     val extra = List.newBuilder[String]
+    var noCache = false
     args.foreach {
       case x: Rule => rules += x.value
       case x: File => files += x.value
       case x: Extra => extra += x.value.foldLeft(x.key)((k, v) => s"$k=$v")
+      case NoCache => noCache = true
     }
-    ShellArgs(rules.result(), files.result(), extra.result())
+    ShellArgs(rules.result(), files.result(), extra.result(), noCache)
   }
 }
 
 case class ShellArgs(
     rules: List[String] = Nil,
     files: List[String] = Nil,
-    extra: List[String] = Nil
+    extra: List[String] = Nil,
+    noCache: Boolean = false
 )