Pass auxiliary class files to zinc so they are deleted together (#3072)

Fixes #3068
`*.tasty`, `*.nir` and `*.sjsir` files were not cleaned up by zinc when
the companion class files were deleted.
This passes the right `AuxiliaryClassFiles` to zinc.

Pull Request: #3072

---------

Co-authored-by: Tobias Roeser <le.petit.fou@web.de>

build.sc

@@ -438,6 +438,18 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima {
     // See https://github.com/com-lihaoyi/mill/pull/2739
     ProblemFilter.exclude[ReversedMissingMethodProblem](
       "mill.scalajslib.ScalaJSModule.mill$scalajslib$ScalaJSModule$$super$scalaLibraryIvyDeps"
+    ),
+    // See https://github.com/com-lihaoyi/mill/pull/3072
+    ProblemFilter.exclude[ReversedMissingMethodProblem](
+      "mill.scalalib.ScalaModule.mill$scalalib$ScalaModule$$super$zincAuxiliaryClassFileExtensions"
+    ),
+    // See https://github.com/com-lihaoyi/mill/pull/3072
+    ProblemFilter.exclude[ReversedMissingMethodProblem](
+      "mill.scalajslib.ScalaJSModule.mill$scalajslib$ScalaJSModule$$super$zincAuxiliaryClassFileExtensions"
+    ),
+    // See https://github.com/com-lihaoyi/mill/pull/3072
+    ProblemFilter.exclude[ReversedMissingMethodProblem](
+      "mill.scalanativelib.ScalaNativeModule.mill$scalanativelib$ScalaNativeModule$$super$zincAuxiliaryClassFileExtensions"
     )
   )
   def mimaPreviousVersions: T[Seq[String]] = Settings.mimaBaseVersions

integration/feature/auxiliary-class-files/repo/app/src/foo.scala

@@ -0,0 +1 @@
+object foo

integration/feature/auxiliary-class-files/repo/build.sc

@@ -0,0 +1,20 @@
+// Issue https://github.com/com-lihaoyi/mill/issues/3068
+import mill._
+import mill.scalalib._
+import mill.scalajslib._
+import mill.scalanativelib._
+
+object app extends Module {
+  trait Common extends ScalaModule {
+    def millSourcePath = super.millSourcePath / os.up
+    def scalaVersion = "3.4.0"
+  }
+
+  object jvm extends Common
+  object js extends Common with ScalaJSModule {
+    def scalaJSVersion = "1.15.0"
+  }
+  object native extends Common with ScalaNativeModule {
+    def scalaNativeVersion = "0.4.17"
+  }
+}

integration/feature/auxiliary-class-files/test/src/AuxiliaryClassFilesTests.scala

@@ -0,0 +1,87 @@
+package mill.integration
+
+import utest._
+
+// Regress test for issue https://github.com/com-lihaoyi/mill/issues/1901
+object AuxiliaryClassFilesTests extends IntegrationTestSuite {
+  val tests: Tests = Tests {
+    test("tasty files are deleted together with companion class files") {
+      initWorkspace()
+      assert(eval("app.jvm.compile"))
+
+      val classes = wd / "out" / "app" / "jvm" / "compile.dest" / "classes"
+      val firstRun = os.list(classes).map(_.last)
+
+      os.remove(wd / "app" / "src" / "foo.scala")
+
+      assert(eval("app.jvm.compile"))
+
+      val secondRun = os.list(classes).map(_.last)
+
+      assert(firstRun == Seq("foo$.class", "foo.class", "foo.tasty"))
+      assert(secondRun == Seq.empty)
+    }
+
+    test("compilation fails when deleting a class used by other files") {
+      initWorkspace()
+
+      os.write(wd / "app" / "src" / "bar.scala", "object bar { println(foo) }")
+      val firstRunSuccessful = eval("app.jvm.compile")
+      assert(firstRunSuccessful)
+
+      val classes = wd / "out" / "app" / "jvm" / "compile.dest" / "classes"
+      val firstRun = os.list(classes).map(_.last)
+
+      os.remove(wd / "app" / "src" / "foo.scala")
+
+      val secondRunSuccessful = eval("app.jvm.compile")
+      assert(!secondRunSuccessful)
+
+      val secondRun = os.list(classes).map(_.last)
+
+      assert(firstRun == Seq(
+        "bar$.class",
+        "bar.class",
+        "bar.tasty",
+        "foo$.class",
+        "foo.class",
+        "foo.tasty"
+      ))
+      assert(secondRun == Seq.empty)
+    }
+
+    test("nir files are deleted together with companion class files") {
+      initWorkspace()
+      assert(eval("app.native.compile"))
+
+      val classes = wd / "out" / "app" / "native" / "compile.dest" / "classes"
+      val firstRun = os.list(classes).map(_.last)
+
+      os.remove(wd / "app" / "src" / "foo.scala")
+
+      assert(eval("app.native.compile"))
+
+      val secondRun = os.list(classes).map(_.last)
+
+      assert(firstRun == Seq("foo$.class", "foo$.nir", "foo.class", "foo.nir", "foo.tasty"))
+      assert(secondRun == Seq.empty)
+    }
+
+    test("sjsir files are deleted together with companion class files") {
+      initWorkspace()
+      assert(eval("app.js.compile"))
+
+      val classes = wd / "out" / "app" / "js" / "compile.dest" / "classes"
+      val firstRun = os.list(classes).map(_.last)
+
+      os.remove(wd / "app" / "src" / "foo.scala")
+
+      assert(eval("app.js.compile"))
+
+      val secondRun = os.list(classes).map(_.last)
+
+      assert(firstRun == Seq("foo$.class", "foo$.sjsir", "foo.class", "foo.tasty"))
+      assert(secondRun == Seq.empty)
+    }
+  }
+}

scalajslib/src/mill/scalajslib/ScalaJSModule.scala

@@ -295,6 +295,9 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer =>
     ))
   }
 
+  override def zincAuxiliaryClassFileExtensions: T[Seq[String]] =
+    super.zincAuxiliaryClassFileExtensions() :+ "sjsir"
+
 }
 
 trait TestScalaJSModule extends ScalaJSModule with TestModule {

scalalib/api/src/mill/scalalib/api/ZincWorkerApi.scala

@@ -62,7 +62,8 @@ trait ZincWorkerApi {
       scalacPluginClasspath: Agg[PathRef],
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean,
-      incrementalCompilation: Boolean
+      incrementalCompilation: Boolean,
+      auxiliaryClassFileExtensions: Seq[String]
   )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
     compileMixed(
       upstreamCompileOutput = upstreamCompileOutput,
@@ -75,8 +76,10 @@ trait ZincWorkerApi {
       compilerClasspath = compilerClasspath,
       scalacPluginClasspath = scalacPluginClasspath,
       reporter = reporter,
-      reportCachedProblems = reportCachedProblems
-    ): @nowarn("cat=deprecation")
+      reportCachedProblems = reportCachedProblems,
+      incrementalCompilation = incrementalCompilation,
+      auxiliaryClassFileExtensions = auxiliaryClassFileExtensions
+    )
 
   /** Compile a mixed Scala/Java or Scala-only project */
   @deprecated("Use override with `incrementalCompilation` parameter", "Mill 0.11.6")
@@ -105,7 +108,40 @@ trait ZincWorkerApi {
       scalacPluginClasspath = scalacPluginClasspath,
       reporter = reporter,
       reportCachedProblems = reportCachedProblems,
-      incrementalCompilation = true
+      incrementalCompilation = true,
+      auxiliaryClassFileExtensions = Seq.empty[String]
+    )
+
+  /** Compile a mixed Scala/Java or Scala-only project */
+  @deprecated("Use override with `auxiliaryClassFileExtensions` parameter", "Mill 0.11.8")
+  def compileMixed(
+      upstreamCompileOutput: Seq[CompilationResult],
+      sources: Agg[os.Path],
+      compileClasspath: Agg[os.Path],
+      javacOptions: Seq[String],
+      scalaVersion: String,
+      scalaOrganization: String,
+      scalacOptions: Seq[String],
+      compilerClasspath: Agg[PathRef],
+      scalacPluginClasspath: Agg[PathRef],
+      reporter: Option[CompileProblemReporter],
+      reportCachedProblems: Boolean,
+      incrementalCompilation: Boolean
+  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+    compileMixed(
+      upstreamCompileOutput = upstreamCompileOutput,
+      sources = sources,
+      compileClasspath = compileClasspath,
+      javacOptions = javacOptions,
+      scalaVersion = scalaVersion,
+      scalaOrganization = scalaOrganization,
+      scalacOptions = scalacOptions,
+      compilerClasspath = compilerClasspath,
+      scalacPluginClasspath = scalacPluginClasspath,
+      reporter = reporter,
+      reportCachedProblems = reportCachedProblems,
+      incrementalCompilation = incrementalCompilation,
+      auxiliaryClassFileExtensions = Seq.empty[String]
     )
 
   def discoverMainClasses(compilationResult: CompilationResult): Seq[String]

scalalib/src/mill/scalalib/JavaModule.scala

@@ -1006,6 +1006,13 @@ trait JavaModule
 
   def forkWorkingDir: T[Path] = T { T.workspace }
 
+  /**
+   * Files extensions that need to be managed by Zinc together with class files.
+   * This means, if zinc needs to remove a class file, it will also remove files
+   * which match the class file basename and a listed file extension.
+   */
+  def zincAuxiliaryClassFileExtensions: T[Seq[String]] = T { Seq.empty[String] }
+
   /**
    * @param all If `true` fetches also source dependencies
    */

scalalib/src/mill/scalalib/ScalaModule.scala

@@ -298,7 +298,8 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
         scalacPluginClasspath = scalacPluginClasspath(),
         reporter = T.reporter.apply(hashCode),
         reportCachedProblems = zincReportCachedProblems(),
-        incrementalCompilation = zincIncrementalCompilation()
+        incrementalCompilation = zincIncrementalCompilation(),
+        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions()
       )
   }
 
@@ -548,6 +549,13 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
     else ZincWorkerUtil.scalaBinaryVersion(scalaVersion())
   }
 
+  override def zincAuxiliaryClassFileExtensions: T[Seq[String]] = T {
+    super.zincAuxiliaryClassFileExtensions() ++ (
+      if (ZincWorkerUtil.isScala3(scalaVersion())) Seq("tasty")
+      else Seq.empty[String]
+    )
+  }
+
   override def artifactSuffix: T[String] = s"${platformSuffix()}_${artifactScalaVersion()}"
 
   override def artifactId: T[String] = artifactName() + artifactSuffix()
@@ -665,7 +673,8 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
         scalacPluginClasspath = semanticDbPluginClasspath(),
         reporter = None,
         reportCachedProblems = zincReportCachedProblems(),
-        incrementalCompilation = zincIncrementalCompilation()
+        incrementalCompilation = zincIncrementalCompilation(),
+        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions()
       )
       .map(r =>
         SemanticDbJavaModule.copySemanticdbFiles(r.classes.path, T.workspace, T.dest / "data")

scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala

@@ -20,6 +20,7 @@ import sbt.internal.util.{ConsoleAppender, ConsoleOut}
 import sbt.mill.SbtLoggerUtils
 import xsbti.compile.{
   AnalysisContents,
+  AuxiliaryClassFileExtension,
   ClasspathOptions,
   CompileAnalysis,
   CompileOrder,
@@ -306,7 +307,8 @@ class ZincWorkerImpl(
       compilers = javaOnlyCompilers(javacOptions),
       reporter = reporter,
       reportCachedProblems = reportCachedProblems,
-      incrementalCompilation = incrementalCompilation
+      incrementalCompilation = incrementalCompilation,
+      auxiliaryClassFileExtensions = Seq.empty[String]
     )
   }
 
@@ -322,7 +324,8 @@ class ZincWorkerImpl(
       scalacPluginClasspath: Agg[PathRef],
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean,
-      incrementalCompilation: Boolean
+      incrementalCompilation: Boolean,
+      auxiliaryClassFileExtensions: Seq[String]
   )(implicit ctx: ZincWorkerApi.Ctx): Result[CompilationResult] = {
     withCompilers(
       scalaVersion = scalaVersion,
@@ -340,7 +343,8 @@ class ZincWorkerImpl(
         compilers = compilers,
         reporter = reporter,
         reportCachedProblems: Boolean,
-        incrementalCompilation
+        incrementalCompilation,
+        auxiliaryClassFileExtensions
       )
     }
   }
@@ -423,7 +427,8 @@ class ZincWorkerImpl(
       compilers: Compilers,
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean,
-      incrementalCompilation: Boolean
+      incrementalCompilation: Boolean,
+      auxiliaryClassFileExtensions: Seq[String]
   )(implicit ctx: ZincWorkerApi.Ctx): Result[CompilationResult] = {
     os.makeDir.all(ctx.dest)
 
@@ -496,6 +501,10 @@ class ZincWorkerImpl(
       .map(path => converter.toVirtualFile(path.toNIO))
       .toArray
 
+    val incOptions = IncOptions.of().withAuxiliaryClassFiles(
+      auxiliaryClassFileExtensions.map(new AuxiliaryClassFileExtension(_)).toArray
+    )
+
     val inputs = ic.inputs(
       classpath = classpath,
       sources = virtualSources,
@@ -512,7 +521,7 @@ class ZincWorkerImpl(
         skip = false,
         cacheFile = zincFile.toNIO,
         cache = new FreshCompilerCache,
-        incOptions = IncOptions.of(),
+        incOptions = incOptions,
         reporter = newReporter,
         progress = None,
         earlyAnalysisStore = None,

scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala

@@ -335,6 +335,9 @@ trait ScalaNativeModule extends ScalaModule { outer =>
     }
   }
 
+  override def zincAuxiliaryClassFileExtensions: T[Seq[String]] =
+    super.zincAuxiliaryClassFileExtensions() :+ "nir"
+
 }
 
 trait TestScalaNativeModule extends ScalaNativeModule with TestModule {