From 479a0ff0c66339a971400bff752b939d1b086532 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 10 Apr 2024 02:33:45 -0400 Subject: [PATCH] Consistent Analysis See also https://github.com/sbt/zinc/pull/1326 This adds a new setting `enableConsistentCompileAnalysis`, which enables the new "Consistent" Analysis format, which is faster and more repeatable than the status quo. This is initialized to `true` by default. It can be opted out either by the setting or using `-Dsbt.analysis2024=false`. --- main/src/main/scala/sbt/Defaults.scala | 57 ++++++++++++------- main/src/main/scala/sbt/Keys.scala | 3 +- .../scala/sbt/internal/AnalysisUtil.scala | 40 +++++++++++++ .../src/main/scala/sbt/internal/SysProp.scala | 2 + .../sbt/internal/server/Definition.scala | 40 ++++++++----- project/Dependencies.scala | 2 +- .../export-jars/changes/B.scala | 7 +-- .../source-dependencies/export-jars/test | 8 +++ .../source-dependencies/implicit-search/test | 2 + .../source-dependencies/linearization/test | 2 + 10 files changed, 121 insertions(+), 42 deletions(-) create mode 100644 main/src/main/scala/sbt/internal/AnalysisUtil.scala diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index ab7e52261db..c1f1723dcb7 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -100,13 +100,7 @@ import scala.xml.NodeSeq // incremental compiler import sbt.SlashSyntax0._ -import sbt.internal.inc.{ - Analysis, - AnalyzingCompiler, - ManagedLoggedReporter, - MixedAnalyzingCompiler, - ScalaInstance -} +import sbt.internal.inc.{ Analysis, AnalyzingCompiler, ManagedLoggedReporter, ScalaInstance } import xsbti.{ CrossValue, VirtualFile, VirtualFileRef } import xsbti.compile.{ AnalysisContents, @@ -875,7 +869,12 @@ object Defaults extends BuildCommon { } def defaultCompileSettings: Seq[Setting[_]] = - globalDefaults(enableBinaryCompileAnalysis := true) + globalDefaults( + Seq( + enableBinaryCompileAnalysis :== true, + enableConsistentCompileAnalysis :== SysProp.analysis2024, + ) + ) lazy val configTasks: Seq[Setting[_]] = docTaskSettings(doc) ++ inTask(compile)(compileInputsSettings) ++ @@ -2299,13 +2298,15 @@ object Defaults extends BuildCommon { */ private[sbt] def compileScalaBackendTask: Initialize[Task[CompileResult]] = Def.task { val setup: Setup = compileIncSetup.value - val useBinary: Boolean = enableBinaryCompileAnalysis.value val analysisResult: CompileResult = compileIncremental.value val exportP = exportPipelining.value // Save analysis midway if pipelining is enabled if (analysisResult.hasModified && exportP) { - val store = - MixedAnalyzingCompiler.staticCachedStore(setup.cacheFile.toPath, !useBinary) + val store = AnalysisUtil.staticCachedStore( + analysisFile = setup.cacheFile.toPath, + useTextAnalysis = !enableBinaryCompileAnalysis.value, + useConsistent = enableConsistentCompileAnalysis.value, + ) val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup()) store.set(contents) // this stores the eary analysis (again) in case the subproject contains a macro @@ -2325,9 +2326,11 @@ object Defaults extends BuildCommon { .debug(s"${name.value}: compileEarly: blocking on earlyOutputPing") earlyOutputPing.await.value }) { - val useBinary: Boolean = enableBinaryCompileAnalysis.value - val store = - MixedAnalyzingCompiler.staticCachedStore(earlyCompileAnalysisFile.value.toPath, !useBinary) + val store = AnalysisUtil.staticCachedStore( + analysisFile = earlyCompileAnalysisFile.value.toPath, + useTextAnalysis = !enableBinaryCompileAnalysis.value, + useConsistent = enableConsistentCompileAnalysis.value, + ) store.get.toOption match { case Some(contents) => contents.getAnalysis case _ => Analysis.empty @@ -2338,13 +2341,15 @@ object Defaults extends BuildCommon { } def compileTask: Initialize[Task[CompileAnalysis]] = Def.task { val setup: Setup = compileIncSetup.value - val useBinary: Boolean = enableBinaryCompileAnalysis.value val c = fileConverter.value // TODO - expose bytecode manipulation phase. val analysisResult: CompileResult = manipulateBytecode.value if (analysisResult.hasModified) { - val store = - MixedAnalyzingCompiler.staticCachedStore(setup.cacheFile.toPath, !useBinary) + val store = AnalysisUtil.staticCachedStore( + analysisFile = setup.cacheFile.toPath, + useTextAnalysis = !enableBinaryCompileAnalysis.value, + useConsistent = enableConsistentCompileAnalysis.value, + ) val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup()) store.set(contents) } @@ -2444,11 +2449,16 @@ object Defaults extends BuildCommon { cachedPerEntryDefinesClassLookup(classpathEntry) } val extra = extraIncOptions.value.map(t2) - val useBinary: Boolean = enableBinaryCompileAnalysis.value val eapath = earlyCompileAnalysisFile.value.toPath val eaOpt = - if (exportPipelining.value) Some(MixedAnalyzingCompiler.staticCachedStore(eapath, !useBinary)) - else None + if (exportPipelining.value) { + val store = AnalysisUtil.staticCachedStore( + analysisFile = eapath, + useTextAnalysis = !enableBinaryCompileAnalysis.value, + useConsistent = enableConsistentCompileAnalysis.value, + ) + Some(store) + } else None Setup.of( lookup, (compile / skip).value, @@ -2538,8 +2548,11 @@ object Defaults extends BuildCommon { def compileAnalysisSettings: Seq[Setting[_]] = Seq( previousCompile := { val setup = compileIncSetup.value - val useBinary: Boolean = enableBinaryCompileAnalysis.value - val store = MixedAnalyzingCompiler.staticCachedStore(setup.cacheFile.toPath, !useBinary) + val store = AnalysisUtil.staticCachedStore( + analysisFile = setup.cacheFile.toPath, + useTextAnalysis = !enableBinaryCompileAnalysis.value, + useConsistent = enableConsistentCompileAnalysis.value, + ) val prev = store.get().toOption match { case Some(contents) => val analysis = Option(contents.getAnalysis).toOptional diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index 69073fc1672..284d9f47215 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -206,7 +206,8 @@ object Keys { val scalaCompilerBridgeSource = settingKey[ModuleID]("Configures the module ID of the sources of the compiler bridge when scalaCompilerBridgeBinaryJar is None").withRank(CSetting) val scalaCompilerBridgeScope = taskKey[Unit]("The compiler bridge scope.").withRank(DTask) val scalaArtifacts = settingKey[Seq[String]]("Configures the list of artifacts which should match the Scala binary version").withRank(CSetting) - val enableBinaryCompileAnalysis = settingKey[Boolean]("Writes the analysis file in binary format") + val enableBinaryCompileAnalysis = settingKey[Boolean]("Writes the analysis file in binary format").withRank(DSetting) + val enableConsistentCompileAnalysis = settingKey[Boolean]("Writes the analysis file in consistent binary format").withRank(DSetting) val crossJavaVersions = settingKey[Seq[String]]("The java versions used during JDK cross testing").withRank(BPlusSetting) val semanticdbEnabled = settingKey[Boolean]("Enables SemanticDB Scalac plugin").withRank(CSetting) val semanticdbCompilerPlugin = settingKey[ModuleID]("SemanticDB Scalac plugin").withRank(CSetting) diff --git a/main/src/main/scala/sbt/internal/AnalysisUtil.scala b/main/src/main/scala/sbt/internal/AnalysisUtil.scala new file mode 100644 index 00000000000..d200ddfde88 --- /dev/null +++ b/main/src/main/scala/sbt/internal/AnalysisUtil.scala @@ -0,0 +1,40 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal + +import java.nio.file.Path +import sbt.internal.inc.MixedAnalyzingCompiler +import scala.concurrent.ExecutionContext +import xsbti.compile.{ AnalysisStore => XAnalysisStore } +import xsbti.compile.analysis.ReadWriteMappers + +private[sbt] object AnalysisUtil { + // some machines have many cores. + // we don't want to occupy them all for analysis serialization. + lazy val parallelism: Int = + scala.math.min( + Runtime.getRuntime.availableProcessors(), + 8, + ) + def staticCachedStore( + analysisFile: Path, + useTextAnalysis: Boolean, + useConsistent: Boolean, + ): XAnalysisStore = + MixedAnalyzingCompiler.staticCachedStore( + analysisFile = analysisFile, + useTextAnalysis = useTextAnalysis, + useConsistent = false, + mappers = ReadWriteMappers.getEmptyMappers(), + sort = true, + ec = ExecutionContext.global, + parallelism = parallelism, + ) +} diff --git a/main/src/main/scala/sbt/internal/SysProp.scala b/main/src/main/scala/sbt/internal/SysProp.scala index bcb2fe8c03a..260c6420ac2 100644 --- a/main/src/main/scala/sbt/internal/SysProp.scala +++ b/main/src/main/scala/sbt/internal/SysProp.scala @@ -139,6 +139,8 @@ object SysProp { def useLog4J: Boolean = getOrFalse("sbt.log.uselog4j") def turbo: Boolean = getOrFalse("sbt.turbo") def pipelining: Boolean = getOrFalse("sbt.pipelining") + // opt-in or out of Zinc's consistent Analysis format. + def analysis2024: Boolean = getOrTrue("sbt.analysis2024") def taskTimings: Boolean = getOrFalse("sbt.task.timings") def taskTimingsOnShutdown: Boolean = getOrFalse("sbt.task.timings.on.shutdown") diff --git a/main/src/main/scala/sbt/internal/server/Definition.scala b/main/src/main/scala/sbt/internal/server/Definition.scala index 4deed738591..669c9dbbdc1 100644 --- a/main/src/main/scala/sbt/internal/server/Definition.scala +++ b/main/src/main/scala/sbt/internal/server/Definition.scala @@ -24,7 +24,7 @@ import sjsonnew.JsonFormat import sjsonnew.shaded.scalajson.ast.unsafe.JValue import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter } -import sbt.internal.inc.{ Analysis, MixedAnalyzingCompiler } +import sbt.internal.inc.Analysis import sbt.internal.inc.JavaInterfaceUtil._ import sbt.internal.protocol.JsonRpcResponseError import sbt.internal.protocol.codec.JsonRPCProtocol @@ -183,11 +183,19 @@ private[sbt] object Definition { } private[this] val AnalysesKey = "lsp.definition.analyses.key" - private[server] type Analyses = Set[((String, Boolean), Option[Analysis])] + private[server] type Analyses = Set[((String, Boolean, Boolean), Option[Analysis])] - private def storeAnalysis(cacheFile: Path, useBinary: Boolean): Option[Analysis] = - MixedAnalyzingCompiler - .staticCachedStore(cacheFile, !useBinary) + private def storeAnalysis( + cacheFile: Path, + useBinary: Boolean, + useConsistent: Boolean, + ): Option[Analysis] = + AnalysisUtil + .staticCachedStore( + analysisFile = cacheFile, + useTextAnalysis = !useBinary, + useConsistent = useConsistent, + ) .get .toOption .map { _.getAnalysis } @@ -195,13 +203,13 @@ private[sbt] object Definition { private[sbt] def updateCache( cache: Cache[String, Analyses] - )(cacheFile: String, useBinary: Boolean): Any = { - cache.get(AnalysesKey, k => Set(cacheFile -> useBinary -> None)) match { + )(cacheFile: String, useBinary: Boolean, useConsistent: Boolean): Any = { + cache.get(AnalysesKey, k => Set((cacheFile, useBinary, useConsistent) -> None)) match { case null => new AnyRef case set => val newSet = set - .filterNot { case ((file, _), _) => file == cacheFile } - .+(cacheFile -> useBinary -> None) + .filterNot { case ((file, _, _), _) => file == cacheFile } + .+((cacheFile, useBinary, useConsistent) -> None) cache.put(AnalysesKey, newSet) } } @@ -221,10 +229,13 @@ private[sbt] object Definition { def collectAnalysesTask = Def.task { val cacheFile: String = compileIncSetup.value.cacheFile.getAbsolutePath - val useBinary = enableBinaryCompileAnalysis.value val s = state.value - s.log.debug(s"analysis location ${cacheFile -> useBinary}") - updateCache(AnalysesAccess.cache)(cacheFile, useBinary) + s.log.debug(s"analysis location ${cacheFile}") + updateCache(AnalysesAccess.cache)( + cacheFile = cacheFile, + useBinary = enableBinaryCompileAnalysis.value, + useConsistent = enableConsistentCompileAnalysis.value, + ) } private[sbt] def getAnalyses: Future[Seq[Analysis]] = { @@ -243,8 +254,9 @@ private[sbt] object Definition { case (_, None) => false } val addToCache = uninitialized.collect { - case (title @ (file, useBinary), _) if Files.exists(Paths.get(file)) => - (title, storeAnalysis(Paths.get(file), !useBinary)) + case (title @ (file, useBinary, useConsistent), _) + if Files.exists(Paths.get(file)) => + (title, storeAnalysis(Paths.get(file), !useBinary, useConsistent)) } val validCaches = working ++ addToCache if (addToCache.nonEmpty) { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6c3c2939f9e..9b1ed11ddd9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -15,7 +15,7 @@ object Dependencies { private val ioVersion = nightlyVersion.getOrElse("1.9.9") private val lmVersion = sys.props.get("sbt.build.lm.version").orElse(nightlyVersion).getOrElse("1.10.0-RC1") - val zincVersion = nightlyVersion.getOrElse("1.10.0-RC1") + val zincVersion = nightlyVersion.getOrElse("1.10.0-RC2") private val sbtIO = "org.scala-sbt" %% "io" % ioVersion diff --git a/sbt-app/src/sbt-test/source-dependencies/export-jars/changes/B.scala b/sbt-app/src/sbt-test/source-dependencies/export-jars/changes/B.scala index 0585279932a..1e08585d1e2 100644 --- a/sbt-app/src/sbt-test/source-dependencies/export-jars/changes/B.scala +++ b/sbt-app/src/sbt-test/source-dependencies/export-jars/changes/B.scala @@ -1,4 +1,3 @@ -object B -{ - def main(args: Array[String]) = assert(args(0).toInt == A.x ) -} \ No newline at end of file +object B { + def main(args: Array[String]) = assert(args(0).toInt == A.x, s"actual A.x is ${A.x}") +} diff --git a/sbt-app/src/sbt-test/source-dependencies/export-jars/test b/sbt-app/src/sbt-test/source-dependencies/export-jars/test index f1a605f00f7..3aff93a7df4 100644 --- a/sbt-app/src/sbt-test/source-dependencies/export-jars/test +++ b/sbt-app/src/sbt-test/source-dependencies/export-jars/test @@ -1,12 +1,16 @@ $ copy-file changes/B.scala B.scala $ copy-file changes/A1.scala a/A.scala +$ sleep 1000 + > run 1 $ copy-file changes/A2.scala a/A.scala +$ sleep 1000 # done this way because last modified times often have ~1s resolution > run 2 $ copy-file changes/A3.scala a/A.scala +$ sleep 1000 > run 3 @@ -14,10 +18,14 @@ $ copy-file changes/build2.sbt build2.sbt > reload $ copy-file changes/A1.scala a/A.scala +$ sleep 1000 + > run 1 $ copy-file changes/A2.scala a/A.scala +$ sleep 1000 > run 2 $ copy-file changes/A3.scala a/A.scala +$ sleep 1000 > run 3 diff --git a/sbt-app/src/sbt-test/source-dependencies/implicit-search/test b/sbt-app/src/sbt-test/source-dependencies/implicit-search/test index 91f01259f09..18d69f6b886 100644 --- a/sbt-app/src/sbt-test/source-dependencies/implicit-search/test +++ b/sbt-app/src/sbt-test/source-dependencies/implicit-search/test @@ -5,5 +5,7 @@ $ copy-file changes/C.scala C.scala -> run $ copy-file changes/A2.scala A.scala +$ sleep 1000 + > compile > run diff --git a/sbt-app/src/sbt-test/source-dependencies/linearization/test b/sbt-app/src/sbt-test/source-dependencies/linearization/test index e6078ebfb1b..22f17664a46 100644 --- a/sbt-app/src/sbt-test/source-dependencies/linearization/test +++ b/sbt-app/src/sbt-test/source-dependencies/linearization/test @@ -5,6 +5,8 @@ # modify D.scala so that the linearization changes $ copy-file changes/D.scala D.scala +$ sleep 1000 + # F.x becomes 11 > compile