Avoid unnecessary dependency downloading by providing fetches per cac…

…he policy (#494) * Avoid unnecessary dependency downloading by providing fetches per cache policy; add ticker logging when they are downloading * Fix GenIdeaTests by making the Log context Option[]al * Add some comments * Rebase and resolve
com-lihaoyi · Dec 20, 2018 · 056cd88 · 056cd88
1 parent 2752dcf
commit 056cd88
Show file tree

Hide file tree

Showing 10 changed files with 128 additions and 21 deletions.
diff --git a/main/core/src/eval/Evaluator.scala b/main/core/src/eval/Evaluator.scala
@@ -251,18 +251,24 @@ case class Evaluator(home: os.Path,
 
     val nonEvaluatedTargets = group.indexed.filterNot(results.contains)
 
-    maybeTargetLabel.foreach { targetLabel =>
+    val tickerPrefix = maybeTargetLabel.map { targetLabel =>
       val inputResults = for {
         target <- nonEvaluatedTargets
         item <- target.inputs.filterNot(group.contains)
       } yield results(item).map(_._1)
 
       val logRun = inputResults.forall(_.isInstanceOf[Result.Success[_]])
 
-      if(logRun) { log.ticker(s"[$counterMsg] $targetLabel ") }
+      val prefix = s"[$counterMsg] $targetLabel "
+      if(logRun) log.ticker(prefix)
+      prefix + "| "
     }
 
-    val multiLogger = resolveLogger(paths.map(_.log))
+    val multiLogger = new ProxyLogger(resolveLogger(paths.map(_.log))) {
+      override def ticker(s: String): Unit = {
+        super.ticker(tickerPrefix.getOrElse("")+s)
+      }
+    }
     var usedDest = Option.empty[(Task[_], Array[StackTraceElement])]
     for (task <- nonEvaluatedTargets) {
       newEvaluated.append(task)

diff --git a/main/core/src/util/Loggers.scala b/main/core/src/util/Loggers.scala
@@ -188,3 +188,21 @@ case class MultiLogger(colored: Boolean, logger1: Logger, logger2: Logger) exten
     logger2.close()
   }
 }
+
+/**
+  * A Logger that forwards all logging to another Logger.  Intended to be
+  * used as a base class for wrappers that modify logging behavior.
+  */
+case class ProxyLogger(logger: Logger) extends Logger {
+  def colored = logger.colored
+
+  lazy val outputStream = logger.outputStream
+  lazy val errorStream = logger.errorStream
+  lazy val inStream = logger.inStream
+
+  def info(s: String) = logger.info(s)
+  def error(s: String) = logger.error(s)
+  def ticker(s: String) = logger.ticker(s)
+  def debug(s: String) = logger.debug(s)
+  override def close() = logger.close()
+}
diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala
@@ -8,7 +8,7 @@ import java.nio.file.attribute.PosixFilePermission
 import java.util.Collections
 import java.util.jar.{JarEntry, JarFile, JarOutputStream}
 
-import coursier.{Cache, Dependency, Fetch, Repository, Resolution}
+import coursier.{Cache, Dependency, Fetch, Repository, Resolution, CachePolicy}
 import coursier.util.{Gather, Task}
 import geny.Generator
 import mill.main.client.InputPumper
@@ -402,10 +402,11 @@ object Jvm {
                           deps: TraversableOnce[coursier.Dependency],
                           force: TraversableOnce[coursier.Dependency],
                           sources: Boolean = false,
-                          mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = {
+                          mapDependencies: Option[Dependency => Dependency] = None,
+                          ctx: Option[mill.util.Ctx.Log] = None): Result[Agg[PathRef]] = {
 
     val (_, resolution) = resolveDependenciesMetadata(
-      repositories, deps, force, mapDependencies
+      repositories, deps, force, mapDependencies, ctx
     )
     val errs = resolution.metadataErrors
     if(errs.nonEmpty) {
@@ -459,7 +460,10 @@ object Jvm {
   def resolveDependenciesMetadata(repositories: Seq[Repository],
                                   deps: TraversableOnce[coursier.Dependency],
                                   force: TraversableOnce[coursier.Dependency],
-                                  mapDependencies: Option[Dependency => Dependency] = None) = {
+                                  mapDependencies: Option[Dependency => Dependency] = None,
+                                  ctx: Option[mill.util.Ctx.Log] = None) = {
+
+    val cachePolicies = CachePolicy.default
 
     val forceVersions = force
       .map(mapDependencies.getOrElse(identity[Dependency](_)))
@@ -472,10 +476,75 @@ object Jvm {
       mapDependencies = mapDependencies
     )
 
-    val fetch = Fetch.from(repositories, Cache.fetch[Task]())
+    val resolutionLogger = ctx.map(c => new TickerResolutionLogger(c))
+    val fetches = cachePolicies.map { p =>
+      Cache.fetch[Task](
+        logger = resolutionLogger,
+        cachePolicy = p
+      )
+    }
+
+    val fetch = Fetch.from(repositories, fetches.head, fetches.tail: _*)
 
     import scala.concurrent.ExecutionContext.Implicits.global
     val resolution = start.process.run(fetch).unsafeRun()
     (deps.toSeq, resolution)
   }
+
+  /**
+    * A Coursier Cache.Logger implementation that updates the ticker with the count and
+    * overall byte size of artifacts being downloaded.
+    *
+    * In practice, this ticket output gets prefixed with the current target for which
+    * dependencies are being resolved, using a ProxyLogger subclass.
+    */
+  class TickerResolutionLogger(ctx: mill.util.Ctx.Log) extends Cache.Logger {
+    case class DownloadState(var current: Long, var total: Long)
+    var downloads = new mutable.TreeMap[String,DownloadState]()
+    var totalDownloadCount = 0
+    var finishedCount = 0
+    var finishedState = DownloadState(0,0)
+
+    def updateTicker(): Unit = {
+      val sums = downloads.values
+        .fold(DownloadState(0,0)) {
+          (s1, s2) => DownloadState(
+            s1.current + s2.current,
+            Math.max(s1.current,s1.total) + Math.max(s2.current,s2.total)
+          )
+        }
+      sums.current += finishedState.current
+      sums.total += finishedState.total
+      ctx.log.ticker(s"Downloading [${downloads.size + finishedCount}/$totalDownloadCount] artifacts (~${sums.current}/${sums.total} bytes)")
+    }
+
+    override def downloadingArtifact(url: String, file: File): Unit = synchronized {
+      totalDownloadCount += 1
+      downloads += url -> DownloadState(0,0)
+      updateTicker()
+    }
+
+    override def downloadLength(url: String, totalLength: Long, alreadyDownloaded: Long, watching: Boolean): Unit = synchronized {
+      val state = downloads(url)
+      state.current = alreadyDownloaded
+      state.total = totalLength
+      updateTicker()
+    }
+
+    override def downloadProgress(url: String, downloaded: Long): Unit = synchronized {
+      val state = downloads(url)
+      state.current = downloaded
+      updateTicker()
+    }
+
+    override def downloadedArtifact(url: String, success: Boolean): Unit = synchronized {
+      val state = downloads(url)
+      finishedState.current += state.current
+      finishedState.total += Math.max(state.current, state.total)
+      finishedCount += 1
+      downloads -= url
+      updateTicker()
+    }
+  }
+
 }
diff --git a/scalajslib/src/ScalaJSModule.scala b/scalajslib/src/ScalaJSModule.scala
@@ -48,7 +48,8 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer =>
     resolveDependencies(
       repositories,
       Lib.depToDependency(_, "2.12.4", ""),
-      commonDeps :+ envDep
+      commonDeps :+ envDep,
+      ctx = Some(implicitly[mill.util.Ctx.Log])
     )
   }
 

diff --git a/scalalib/src/GenIdeaImpl.scala b/scalalib/src/GenIdeaImpl.scala
@@ -42,7 +42,7 @@ object GenIdeaImpl {
 
     val evaluator = new Evaluator(ctx.home, os.pwd / 'out, os.pwd / 'out, rootModule, ctx.log)
 
-    for((relPath, xml) <- xmlFileLayout(evaluator, rootModule, jdkInfo)){
+    for((relPath, xml) <- xmlFileLayout(evaluator, rootModule, jdkInfo, Some(ctx))){
       os.write.over(os.pwd/relPath, pp.format(xml), createFolders = true)
     }
   }
@@ -61,6 +61,7 @@ object GenIdeaImpl {
   def xmlFileLayout(evaluator: Evaluator,
                     rootModule: mill.Module,
                     jdkInfo: (String,String),
+                    ctx: Option[Log],
                     fetchMillModules: Boolean = true): Seq[(os.RelPath, scala.xml.Node)] = {
 
     val modules = rootModule.millInternal.segmentsToModules.values
@@ -78,7 +79,10 @@ object GenIdeaImpl {
             repos.toList,
             Lib.depToDependency(_, "2.12.4", ""),
             for(name <- artifactNames)
-            yield ivy"com.lihaoyi::mill-$name:${sys.props("MILL_VERSION")}"
+            yield ivy"com.lihaoyi::mill-$name:${sys.props("MILL_VERSION")}",
+            false,
+            None,
+            ctx
           )
           res.items.toList.map(_.path)
       }

diff --git a/scalalib/src/JavaModule.scala b/scalalib/src/JavaModule.scala
@@ -138,7 +138,8 @@ trait JavaModule extends mill.Module with TaskModule { outer =>
       resolveCoursierDependency().apply(_),
       deps(),
       sources,
-      mapDependencies = Some(mapDependencies())
+      mapDependencies = Some(mapDependencies()),
+      Some(implicitly[mill.util.Ctx.Log])
     )
   }
 

diff --git a/scalalib/src/Lib.scala b/scalalib/src/Lib.scala
@@ -35,13 +35,15 @@ object Lib{
   def resolveDependenciesMetadata(repositories: Seq[Repository],
                                   depToDependency: Dep => coursier.Dependency,
                                   deps: TraversableOnce[Dep],
-                                  mapDependencies: Option[Dependency => Dependency] = None) = {
+                                  mapDependencies: Option[Dependency => Dependency] = None,
+                                  ctx: Option[mill.util.Ctx.Log] = None) = {
     val depSeq = deps.toSeq
     mill.modules.Jvm.resolveDependenciesMetadata(
       repositories,
       depSeq.map(depToDependency),
       depSeq.filter(_.force).map(depToDependency),
-      mapDependencies
+      mapDependencies,
+      ctx
     )
   }
   /**
@@ -55,14 +57,16 @@ object Lib{
                           depToDependency: Dep => coursier.Dependency,
                           deps: TraversableOnce[Dep],
                           sources: Boolean = false,
-                          mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = {
+                          mapDependencies: Option[Dependency => Dependency] = None,
+                          ctx: Option[mill.util.Ctx.Log] = None): Result[Agg[PathRef]] = {
     val depSeq = deps.toSeq
     mill.modules.Jvm.resolveDependencies(
       repositories,
       depSeq.map(depToDependency),
       depSeq.filter(_.force).map(depToDependency),
       sources,
-      mapDependencies
+      mapDependencies,
+      ctx
     )
   }
   def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String) =

diff --git a/scalalib/src/ZincWorkerModule.scala b/scalalib/src/ZincWorkerModule.scala
@@ -102,7 +102,8 @@ trait ZincWorkerModule extends mill.Module{
     resolveDependencies(
       repositories,
       Lib.depToDependency(_, "2.12.4", ""),
-      Seq(ivy"org.scala-sbt:compiler-interface:${Versions.zinc}")
+      Seq(ivy"org.scala-sbt:compiler-interface:${Versions.zinc}"),
+      ctx = Some(implicitly[mill.util.Ctx.Log])
     )
   }
 

diff --git a/scalalib/test/src/GenIdeaTests.scala b/scalalib/test/src/GenIdeaTests.scala
@@ -28,7 +28,7 @@ object GenIdeaTests extends TestSuite {
       val layout = GenIdeaImpl.xmlFileLayout(
         helloWorldEvaluator.evaluator,
         HelloWorld,
-        ("JDK_1_8", "1.8 (1)"), fetchMillModules = false)
+        ("JDK_1_8", "1.8 (1)"), None, fetchMillModules = false)
       for((relPath, xml) <- layout){
         os.write.over(millSourcePath/ "generated"/ relPath, pp.format(xml), createFolders = true)
       }

diff --git a/scalanativelib/src/ScalaNativeModule.scala b/scalanativelib/src/ScalaNativeModule.scala
@@ -52,7 +52,8 @@ trait ScalaNativeModule extends ScalaModule { outer =>
       Lib.resolveDependencies(
         Seq(Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")),
         Lib.depToDependency(_, "2.12.4", ""),
-        Seq(ivy"com.lihaoyi::mill-scalanativelib-worker-${scalaNativeBinaryVersion()}:${sys.props("MILL_VERSION")}")
+        Seq(ivy"com.lihaoyi::mill-scalanativelib-worker-${scalaNativeBinaryVersion()}:${sys.props("MILL_VERSION")}"),
+        ctx = Some(implicitly[mill.util.Ctx.Log])
       )
   }
 
@@ -83,7 +84,8 @@ trait ScalaNativeModule extends ScalaModule { outer =>
     Lib.resolveDependencies(
       Seq(Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")),
       Lib.depToDependency(_, scalaVersion(), platformSuffix()),
-      toolsIvyDeps()
+      toolsIvyDeps(),
+      ctx = Some(implicitly[mill.util.Ctx.Log])
     ).map(t => (scalaNativeWorkerClasspath().toSeq ++ t.toSeq).map(_.path))
   }
 
@@ -200,7 +202,8 @@ trait TestScalaNativeModule extends ScalaNativeModule with TestModule { testOute
     Lib.resolveDependencies(
       repositories,
       Lib.depToDependency(_, scalaVersion(), ""),
-      transitiveIvyDeps().filter(d => d.cross.isBinary && supportedTestFrameworks(d.dep.module.name))
+      transitiveIvyDeps().filter(d => d.cross.isBinary && supportedTestFrameworks(d.dep.module.name)),
+      ctx = Some(implicitly[mill.util.Ctx.Log])
     )
   }