From 8cdff6070d85fc7a502b66b9e487e76559c87281 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 13:20:06 +0800 Subject: [PATCH 01/13] . --- main/eval/src/mill/eval/EvaluatorCore.scala | 143 +++++++++++--------- main/src/mill/main/MainModule.scala | 8 +- runner/src/mill/runner/MillMain.scala | 6 +- 3 files changed, 82 insertions(+), 75 deletions(-) diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 0a6a1296165..b32bcafa088 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -66,93 +66,104 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { reporter: Int => Option[CompileProblemReporter] = _ => Option.empty[CompileProblemReporter], testReporter: TestReporter = DummyTestReporter, ec: ExecutionContext with AutoCloseable, - contextLoggerMsg: Int => String + contextLoggerMsg0: Int => String ): Evaluator.Results = { - implicit val implicitEc = ec - os.makeDir.all(outPath) val chromeProfileLogger = new ChromeProfileLogger(outPath / "mill-chrome-profile.json") val profileLogger = new ProfileLogger(outPath / "mill-profile.json") val threadNumberer = new ThreadNumberer() val (sortedGroups, transitive) = Plan.plan(goals) val interGroupDeps = findInterGroupDeps(sortedGroups) - val terminals = sortedGroups.keys().toVector + val terminals0 = sortedGroups.keys().toVector val failed = new AtomicBoolean(false) val count = new AtomicInteger(1) val futures = mutable.Map.empty[Terminal, Future[Option[GroupEvaluator.Results]]] - // We walk the task graph in topological order and schedule the futures - // to run asynchronously. During this walk, we store the scheduled futures - // in a dictionary. When scheduling each future, we are guaranteed that the - // necessary upstream futures will have already been scheduled and stored, - // due to the topological order of traversal. - for (terminal <- terminals) { - val deps = interGroupDeps(terminal) - futures(terminal) = Future.sequence(deps.map(futures)).map { upstreamValues => - if (failed.get()) None - else { - val upstreamResults = upstreamValues - .iterator - .flatMap(_.iterator.flatMap(_.newResults)) - .toMap - - val startTime = System.nanoTime() / 1000 - val threadId = threadNumberer.getThreadId(Thread.currentThread()) - val counterMsg = s"${count.getAndIncrement()}/${terminals.size}" - val contextLogger = PrefixLogger( - out = logger, - context = contextLoggerMsg(threadId), - tickerContext = GroupEvaluator.dynamicTickerPrefix.value - ) - - val res = evaluateGroupCached( - terminal = terminal, - group = sortedGroups.lookupKey(terminal), - results = upstreamResults, - counterMsg = counterMsg, - zincProblemReporter = reporter, - testReporter = testReporter, - logger = contextLogger - ) - - if (failFast && res.newResults.values.exists(_.result.asSuccess.isEmpty)) - failed.set(true) - - val endTime = System.nanoTime() / 1000 - - val duration = endTime - startTime - - chromeProfileLogger.log( - task = Terminal.printTerm(terminal), - cat = "job", - startTime = startTime, - duration = duration, - threadId = threadNumberer.getThreadId(Thread.currentThread()), - cached = res.cached - ) - - profileLogger.log( - ProfileLogger.Timing( - terminal.render, - (duration / 1000).toInt, - res.cached, - deps.map(_.render), - res.inputsHash, - res.previousInputsHash + def evaluateTerminals(terminals: Seq[Terminal], + contextLoggerMsg: Int => String)(implicit ec: ExecutionContext) = { + // We walk the task graph in topological order and schedule the futures + // to run asynchronously. During this walk, we store the scheduled futures + // in a dictionary. When scheduling each future, we are guaranteed that the + // necessary upstream futures will have already been scheduled and stored, + // due to the topological order of traversal. + for (terminal <- terminals) { + val deps = interGroupDeps(terminal) + futures(terminal) = Future.sequence(deps.map(futures)).map { upstreamValues => + if (failed.get()) None + else { + val upstreamResults = upstreamValues + .iterator + .flatMap(_.iterator.flatMap(_.newResults)) + .toMap + + val startTime = System.nanoTime() / 1000 + val threadId = threadNumberer.getThreadId(Thread.currentThread()) + val counterMsg = s"${count.getAndIncrement()}/${terminals.size}" + val contextLogger = PrefixLogger( + out = logger, + context = contextLoggerMsg(threadId), + tickerContext = GroupEvaluator.dynamicTickerPrefix.value + ) + + val res = evaluateGroupCached( + terminal = terminal, + group = sortedGroups.lookupKey(terminal), + results = upstreamResults, + counterMsg = counterMsg, + zincProblemReporter = reporter, + testReporter = testReporter, + logger = contextLogger + ) + + if (failFast && res.newResults.values.exists(_.result.asSuccess.isEmpty)) + failed.set(true) + + val endTime = System.nanoTime() / 1000 + + val duration = endTime - startTime + + chromeProfileLogger.log( + task = Terminal.printTerm(terminal), + cat = "job", + startTime = startTime, + duration = duration, + threadId = threadNumberer.getThreadId(Thread.currentThread()), + cached = res.cached + ) + + profileLogger.log( + ProfileLogger.Timing( + terminal.render, + (duration / 1000).toInt, + res.cached, + deps.map(_.render), + res.inputsHash, + res.previousInputsHash + ) ) - ) - Some(res) + Some(res) + } } } } - val finishedOptsMap = terminals + val (commands, tasks) = terminals0.partition { + case Terminal.Labelled(c: Command[_], _) => true + case _ => false + } + + // Run all non-command tasks according to the threads + // given but run the commands in linear order + evaluateTerminals(tasks, contextLoggerMsg0)(ec) + evaluateTerminals(commands, _ => "")(ExecutionContexts.RunNow) + + val finishedOptsMap = terminals0 .map(t => (t, Await.result(futures(t), duration.Duration.Inf))) .toMap - val results0: Vector[(Task[_], TaskResult[(Val, Int)])] = terminals + val results0: Vector[(Task[_], TaskResult[(Val, Int)])] = terminals0 .flatMap { t => sortedGroups.lookupKey(t).flatMap { t0 => finishedOptsMap(t) match { diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index 308ef993713..177c1d00f81 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -117,7 +117,7 @@ trait MainModule extends mill.define.Module { case Left(err) => Result.Failure(err) case Right(resolvedSegmentsList) => val resolvedStrings = resolvedSegmentsList.map(_.render) - resolvedStrings.sorted.foreach(Target.log.outputStream.println) + resolvedStrings.sorted.foreach(evaluator.baseLogger.outputStream.println) Result.Success(resolvedStrings) } } @@ -131,7 +131,7 @@ trait MainModule extends mill.define.Module { case Left(err) => Result.Failure(err) case Right(success) => val renderedTasks = success.map(_.segments.render) - renderedTasks.foreach(Target.log.outputStream.println) + renderedTasks.foreach(evaluator.baseLogger.outputStream.println) Result.Success(renderedTasks) } } @@ -189,7 +189,7 @@ trait MainModule extends mill.define.Module { val labels = list .collect { case n: NamedTask[_] => n.ctx.segments.render } - labels.foreach(Target.log.outputStream.println(_)) + labels.foreach(evaluator.baseLogger.outputStream.println(_)) Result.Success(labels) } @@ -307,7 +307,7 @@ trait MainModule extends mill.define.Module { for { str <- truncated ++ Iterator("\n") } sb.append(str) sb.toString() }).mkString("\n") - Target.log.outputStream.println(output) + evaluator.baseLogger.outputStream.println(output) fansi.Str(output).plainText } } diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index 2059ca3c164..b8a202eb30a 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -180,11 +180,7 @@ object MillMain { val userSpecifiedProperties = userSpecifiedProperties0 ++ config.extraSystemProperties - val threadCount = config.threadCountRaw match { - case None => Some(1) - case Some(0) => None - case Some(n) => Some(n) - } + val threadCount = config.threadCountRaw if (mill.main.client.Util.isJava9OrAbove) { val rt = config.home / Export.rtJarName From cc29c49ca3dd045fa5fb28c0dfffd160a0ec2ca1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 13:21:30 +0800 Subject: [PATCH 02/13] . --- main/eval/src/mill/eval/EvaluatorCore.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index b32bcafa088..aa3a1c26899 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -80,8 +80,9 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val futures = mutable.Map.empty[Terminal, Future[Option[GroupEvaluator.Results]]] - def evaluateTerminals(terminals: Seq[Terminal], - contextLoggerMsg: Int => String)(implicit ec: ExecutionContext) = { + def evaluateTerminals(terminals: Seq[Terminal], contextLoggerMsg: Int => String)(implicit + ec: ExecutionContext + ) = { // We walk the task graph in topological order and schedule the futures // to run asynchronously. During this walk, we store the scheduled futures // in a dictionary. When scheduling each future, we are guaranteed that the From 06da986fcc721c4318567c894296a33d0aa2053a Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 13:26:10 +0800 Subject: [PATCH 03/13] . --- main/src/mill/main/MainModule.scala | 8 ++++---- runner/src/mill/runner/MillMain.scala | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index 177c1d00f81..308ef993713 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -117,7 +117,7 @@ trait MainModule extends mill.define.Module { case Left(err) => Result.Failure(err) case Right(resolvedSegmentsList) => val resolvedStrings = resolvedSegmentsList.map(_.render) - resolvedStrings.sorted.foreach(evaluator.baseLogger.outputStream.println) + resolvedStrings.sorted.foreach(Target.log.outputStream.println) Result.Success(resolvedStrings) } } @@ -131,7 +131,7 @@ trait MainModule extends mill.define.Module { case Left(err) => Result.Failure(err) case Right(success) => val renderedTasks = success.map(_.segments.render) - renderedTasks.foreach(evaluator.baseLogger.outputStream.println) + renderedTasks.foreach(Target.log.outputStream.println) Result.Success(renderedTasks) } } @@ -189,7 +189,7 @@ trait MainModule extends mill.define.Module { val labels = list .collect { case n: NamedTask[_] => n.ctx.segments.render } - labels.foreach(evaluator.baseLogger.outputStream.println(_)) + labels.foreach(Target.log.outputStream.println(_)) Result.Success(labels) } @@ -307,7 +307,7 @@ trait MainModule extends mill.define.Module { for { str <- truncated ++ Iterator("\n") } sb.append(str) sb.toString() }).mkString("\n") - evaluator.baseLogger.outputStream.println(output) + Target.log.outputStream.println(output) fansi.Str(output).plainText } } diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index b8a202eb30a..fd803c546d9 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -180,7 +180,11 @@ object MillMain { val userSpecifiedProperties = userSpecifiedProperties0 ++ config.extraSystemProperties - val threadCount = config.threadCountRaw + val threadCount = config.threadCountRaw match { + case None => None + case Some(0) => None + case Some(n) => Some(n) + } if (mill.main.client.Util.isJava9OrAbove) { val rt = config.home / Export.rtJarName From 00b9deab10d340a786f9b58e02af529ee32c20cc Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 14:07:22 +0800 Subject: [PATCH 04/13] . --- main/eval/src/mill/eval/EvaluatorCore.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index aa3a1c26899..6fcf7984861 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -157,7 +157,11 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { // Run all non-command tasks according to the threads // given but run the commands in linear order - evaluateTerminals(tasks, contextLoggerMsg0)(ec) + evaluateTerminals( + tasks, + if (sys.env.contains("MILL_TEST_SUITE")) _ => "" + else contextLoggerMsg0 + )(ec) evaluateTerminals(commands, _ => "")(ExecutionContexts.RunNow) val finishedOptsMap = terminals0 From d54e932a717481ef4ba4fbe1d874837d2b45af6b Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 14:31:58 +0800 Subject: [PATCH 05/13] . --- .../test/src/CodeSigNestedTests.scala | 36 +++++++++---------- .../test/src/CodeSigScalaModuleTests.scala | 26 +++++++------- .../test/src/ScriptsInvalidationTests.scala | 22 ++++++------ 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/integration/feature/codesig-nested/test/src/CodeSigNestedTests.scala b/integration/feature/codesig-nested/test/src/CodeSigNestedTests.scala index e228fb82657..a80371267f6 100644 --- a/integration/feature/codesig-nested/test/src/CodeSigNestedTests.scala +++ b/integration/feature/codesig-nested/test/src/CodeSigNestedTests.scala @@ -14,7 +14,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { // with no changes val initial = evalStdout("outer.inner.qux") assert( - initial.out.linesIterator.toSeq == Seq( + initial.out.linesIterator.toSet == Set( "running foo", "running helperFoo", "running bar", @@ -32,7 +32,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("running foo", "running foo2")) val mangledFoo = evalStdout("outer.inner.qux") assert( - mangledFoo.out.linesIterator.toSeq == Seq( + mangledFoo.out.linesIterator.toSet == Set( "running foo2", "running helperFoo" // The return value of foo did not change so qux is not invalidated @@ -42,7 +42,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("; helperFoo }", "; helperFoo + 4 }")) val mangledHelperFooCall = evalStdout("outer.inner.qux") assert( - mangledHelperFooCall.out.linesIterator.toSeq == Seq( + mangledHelperFooCall.out.linesIterator.toSet == Set( "running foo2", "running helperFoo", // The return value of foo changes from 1 to 1+4=5, so qux is invalidated @@ -54,9 +54,9 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("running qux", "running qux2")) val mangledQux = evalStdout("outer.inner.qux") assert( - mangledQux.out.linesIterator.toSeq == + mangledQux.out.linesIterator.toSet == // qux itself was changed, and so it is invalidated - Seq("running qux2", "running helperQux") + Set("running qux2", "running helperQux") ) // Changing the body of some helper method that gets called by a T{...} @@ -65,7 +65,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace(" 1 ", " 6 ")) val mangledHelperFooValue = evalStdout("outer.inner.qux") assert( - mangledHelperFooValue.out.linesIterator.toSeq == Seq( + mangledHelperFooValue.out.linesIterator.toSet == Set( "running foo2", "running helperFoo", // Because the return value of helperFoo/foo changes from 1+4=5 to 6+5=11, qux is invalidated @@ -77,7 +77,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("running helperBar", "running helperBar2")) val mangledHelperBar = evalStdout("outer.inner.qux") assert( - mangledHelperBar.out.linesIterator.toSeq == Seq( + mangledHelperBar.out.linesIterator.toSet == Set( "running bar", "running helperBar2" // We do not need to re-evaluate qux because the return value of bar did not change @@ -87,7 +87,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("20", "70")) val mangledHelperBarValue = evalStdout("outer.inner.qux") assert( - mangledHelperBarValue.out.linesIterator.toSeq == Seq( + mangledHelperBarValue.out.linesIterator.toSet == Set( "running bar", "running helperBar2", // Because the return value of helperBar/bar changes from 20 to 70, qux is invalidated @@ -99,9 +99,9 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("running helperQux", "running helperQux2")) val mangledBar = evalStdout("outer.inner.qux") assert( - mangledBar.out.linesIterator.toSeq == + mangledBar.out.linesIterator.toSet == // helperQux was changed, so qux needs to invalidate - Seq("running qux2", "running helperQux2") + Set("running qux2", "running helperQux2") ) // Make sure changing `val`s in varying levels of nested modules conservatively invalidates @@ -109,7 +109,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("val valueFoo = 0", "val valueFoo = 10")) val mangledValFoo = evalStdout("outer.inner.qux") assert( - mangledValFoo.out.linesIterator.toSeq == Seq( + mangledValFoo.out.linesIterator.toSet == Set( "running foo2", "running helperFoo", "running bar", @@ -122,7 +122,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("val valueBar = 0", "val valueBar = 10")) val mangledValBar = evalStdout("outer.inner.qux") assert( - mangledValBar.out.linesIterator.toSeq == Seq( + mangledValBar.out.linesIterator.toSet == Set( "running bar", "running helperBar2", "running qux2", @@ -133,7 +133,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { mangleFile(wsRoot / "build.sc", _.replace("val valueQux = 0", "val valueQux = 10")) val mangledValQux = evalStdout("outer.inner.qux") assert( - mangledValQux.out.linesIterator.toSeq == Seq( + mangledValQux.out.linesIterator.toSet == Set( "running qux2", "running helperQux2" ) @@ -145,7 +145,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { ) val mangledValFooUsedInBar = evalStdout("outer.inner.qux") assert( - mangledValFooUsedInBar.out.linesIterator.toSeq == Seq( + mangledValFooUsedInBar.out.linesIterator.toSet == Set( "running foo2", "running helperFoo", "running bar", @@ -161,7 +161,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { ) val mangledValBarUsedInQux = evalStdout("outer.inner.qux") assert( - mangledValBarUsedInQux.out.linesIterator.toSeq == Seq( + mangledValBarUsedInQux.out.linesIterator.toSet == Set( "running bar", "running helperBar2", "running qux2", @@ -191,7 +191,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { "trait" - { val initial = evalStdout("traitOuter.traitInner.inner") assert( - initial.out.linesIterator.toSeq == Seq( + initial.out.linesIterator.toSet == Set( "running foo", "running helperFoo", "running outer", @@ -210,7 +210,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { ) val mangleTraitInnerValue = evalStdout("traitOuter.traitInner.inner") assert( - mangleTraitInnerValue.out.linesIterator.toSeq == Seq( + mangleTraitInnerValue.out.linesIterator.toSet == Set( "running inner", "running helperTraitInner" ) @@ -222,7 +222,7 @@ object CodeSigNestedTests extends IntegrationTestSuite { ) val mangleTraitOuterValue = evalStdout("traitOuter.traitInner.inner") assert( - mangleTraitOuterValue.out.linesIterator.toSeq == Seq( + mangleTraitOuterValue.out.linesIterator.toSet == Set( "running outer", "running helperTraitOuter", "running inner", diff --git a/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala b/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala index 45f38bca99f..4ac75532dbc 100644 --- a/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala +++ b/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala @@ -5,7 +5,7 @@ import utest._ object CodeSigScalaModuleTests extends IntegrationTestSuite { val tests: Tests = Tests { def filterLines(out: String) = { - out.linesIterator.filter(!_.contains("[info]")).toSeq + out.linesIterator.filter(!_.contains("[info]")).toSet } val wsRoot = initWorkspace() "single" - { @@ -17,7 +17,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { assert( filterLines(initial.out) == - Seq( + Set( "Foo generating sources...", "Foo compiling...", "Foo Hello World", @@ -28,7 +28,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { val cached = evalStdout("foo.run") assert( filterLines(cached.out) == - Seq( + Set( "Foo Hello World", "Foo running..." ) @@ -38,7 +38,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { // and any downstream targets mangleFile(wsRoot / "build.sc", _.replace("Foo running...", "FOO RUNNING")) val mangledFoo = evalStdout("foo.run") - assert(filterLines(mangledFoo.out) == Seq("Foo Hello World", "FOO RUNNING")) + assert(filterLines(mangledFoo.out) == Set("Foo Hello World", "FOO RUNNING")) // Changing the body `foo.compile` invalidates `foo.compile`, and downstream // `foo.run` runs regardless @@ -47,7 +47,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { assert( filterLines(mangledFoo2.out) == - Seq( + Set( "FOO COMPILING", "Foo Hello World", "FOO RUNNING" @@ -64,7 +64,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { assert( filterLines(mangledFoo3.out) == - Seq( + Set( "FOO GENERATING SOURCES", "Foo Hello World", "FOO RUNNING" @@ -78,7 +78,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { assert( filterLines(mangledFoo4.out) == - Seq( + Set( "FOO GENERATING SOURCES", "FOO COMPILING", "Foo HELLO WORLD", @@ -91,7 +91,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { assert( filterLines(mangledFoo5.out) == - Seq( + Set( "FOO COMPILING", "Foo HELLO WORLD", "FOO RUNNING" @@ -111,7 +111,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { val mangledFoo6 = evalStdout("foo.run") assert( filterLines(mangledFoo6.out) == - Seq( + Set( "Foo HELLO WORLD", "FOO RUNNING" ) @@ -128,7 +128,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { assert( filterLines(initial.out) == - Seq( + Set( "Foo generating sources...", "Foo compiling...", "Foo assembly...", @@ -154,14 +154,14 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { // evaluation can see the return value was not changed and avoid invalidation mangleFile(wsRoot / "build.sc", _.replace("Foo compiling...", "FOO COMPILING")) val mangledFoo2 = evalStdout("{foo,bar,qux}.assembly") - assert(filterLines(mangledFoo2.out) == Seq("FOO COMPILING")) + assert(filterLines(mangledFoo2.out) == Set("FOO COMPILING")) mangleFile( wsRoot / "build.sc", _.replace("Foo generating sources...", "FOO generating sources") ) val mangledFoo3 = evalStdout("{foo,bar,qux}.assembly") - assert(filterLines(mangledFoo3.out) == Seq("FOO generating sources")) + assert(filterLines(mangledFoo3.out) == Set("FOO generating sources")) // Changing the implementation of foo.generatedSources in a way that changes // its return value does cause downstream targets in foo and bar to invalidate, @@ -174,7 +174,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { assert( filterLines(mangledFoo4.out) == - Seq( + Set( "FOO generating sources", "FOO COMPILING", "Foo assembly...", diff --git a/integration/feature/invalidation/test/src/ScriptsInvalidationTests.scala b/integration/feature/invalidation/test/src/ScriptsInvalidationTests.scala index c376a31b6c9..f9bc150bf5d 100644 --- a/integration/feature/invalidation/test/src/ScriptsInvalidationTests.scala +++ b/integration/feature/invalidation/test/src/ScriptsInvalidationTests.scala @@ -4,10 +4,10 @@ import utest._ object ScriptsInvalidationTests extends IntegrationTestSuite { - def runTask(task: String): Vector[String] = { + def runTask(task: String): Set[String] = { val res = evalStdout(task) assert(res.isSuccess) - res.out.linesIterator.map(_.trim).toVector + res.out.linesIterator.map(_.trim).toSet } val tests: Tests = Tests { @@ -17,7 +17,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { val result = runTask("task") - val expected = Seq("a", "d", "b", "c") + val expected = Set("a", "d", "b", "c") assert(result == expected) } @@ -39,7 +39,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { initWorkspace() val result = runTask("task") - val expected = Seq("a", "d", "b", "c") + val expected = Set("a", "d", "b", "c") assert(result == expected) } @@ -51,7 +51,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { os.write.over(workspacePath / inputD, newContent) val result = runTask("task") - val expected = Seq("d2", "b") + val expected = Set("d2", "b") assert(result == expected) } @@ -61,7 +61,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { initWorkspace() val result = runTask("module.task") - val expected = Seq("a", "d", "b", "c", "task") + val expected = Set("a", "d", "b", "c", "task") assert(result == expected) } @@ -73,7 +73,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { os.write.over(workspacePath / buildPath, newContent) val result = runTask("module.task") - val expected = Seq("task2") + val expected = Set("task2") assert(result == expected) } @@ -83,7 +83,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { initWorkspace() val result = runTask("taskE") - val expected = Seq("a", "e", "taskE") + val expected = Set("a", "e", "taskE") assert(result == expected) } @@ -95,7 +95,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { os.write.over(workspacePath / buildPath, newContent) val result = runTask("taskE") - val expected = Seq("taskE2") + val expected = Set("taskE2") assert(result == expected) } @@ -104,7 +104,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { initWorkspace() val result = runTask("taskSymbols") - val expected = Seq("taskSymbols") + val expected = Set("taskSymbols") assert(result == expected) } @@ -112,7 +112,7 @@ object ScriptsInvalidationTests extends IntegrationTestSuite { initWorkspace() val result = runTask("taskSymbolsInFile") - val expected = Seq("taskSymbolsInFile") + val expected = Set("taskSymbolsInFile") assert(result == expected) } From 13907e46cb5c6029ab6c312cb2fe468f3d33a341 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 14:52:22 +0800 Subject: [PATCH 06/13] . --- .../codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala b/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala index 4ac75532dbc..ad19e563aab 100644 --- a/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala +++ b/integration/feature/codesig-scalamodule/test/src/CodeSigScalaModuleTests.scala @@ -142,7 +142,7 @@ object CodeSigScalaModuleTests extends IntegrationTestSuite { ) val cached = evalStdout("{foo,bar,qux}.assembly") - assert(filterLines(cached.out) == Seq()) + assert(filterLines(cached.out) == Set()) // Changing the implementation of foo.compile or foo.generatedSources // without changing its return value causes that specific target to From 27ab50a585ee2264937505a36f9194439169ade0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 16:17:28 +0800 Subject: [PATCH 07/13] TEST_DEST_FOLDER --- main/test/src/mill/util/TestUtil.scala | 2 +- scalalib/src/mill/scalalib/TestModule.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main/test/src/mill/util/TestUtil.scala b/main/test/src/mill/util/TestUtil.scala index b8a27538315..870f82862a1 100644 --- a/main/test/src/mill/util/TestUtil.scala +++ b/main/test/src/mill/util/TestUtil.scala @@ -12,7 +12,7 @@ import scala.collection.mutable object TestUtil extends MillTestKit { - override val targetDir = os.pwd / "target" + override val targetDir = os.Path(sys.env("MILL_TEST_DEST_FOLDER")) def getOutPath()(implicit fullName: sourcecode.FullName, tp: TestPath): os.Path = { getOutPath(tp.value) diff --git a/scalalib/src/mill/scalalib/TestModule.scala b/scalalib/src/mill/scalalib/TestModule.scala index 4b539b463aa..066d824c139 100644 --- a/scalalib/src/mill/scalalib/TestModule.scala +++ b/scalalib/src/mill/scalalib/TestModule.scala @@ -180,7 +180,7 @@ trait TestModule _.path ), jvmArgs = jvmArgs, - envArgs = forkEnv(), + envArgs = Map("MILL_TEST_DEST_FOLDER" -> T.dest.toString()) ++ forkEnv(), mainArgs = mainArgs, workingDir = forkWorkingDir(), useCpPassingJar = useArgsFile From 4c3244b5b2a62b6dcb4a3b3427da70129f6164f4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 16:28:45 +0800 Subject: [PATCH 08/13] TEST_DEST_FOLDER --- main/test/src/mill/util/TestUtil.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main/test/src/mill/util/TestUtil.scala b/main/test/src/mill/util/TestUtil.scala index 870f82862a1..ebe832f3ccc 100644 --- a/main/test/src/mill/util/TestUtil.scala +++ b/main/test/src/mill/util/TestUtil.scala @@ -12,7 +12,10 @@ import scala.collection.mutable object TestUtil extends MillTestKit { - override val targetDir = os.Path(sys.env("MILL_TEST_DEST_FOLDER")) + override val targetDir = sys.env.get("MILL_TEST_DEST_FOLDER") match{ + case Some(v) => os.Path(v) + case None => os.pwd / "target" + } def getOutPath()(implicit fullName: sourcecode.FullName, tp: TestPath): os.Path = { getOutPath(tp.value) From 21c85d7834ee8ac12066d27b0a301bdbca9482fc Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Jul 2024 20:32:57 +0800 Subject: [PATCH 09/13] . --- main/test/src/mill/util/TestUtil.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/test/src/mill/util/TestUtil.scala b/main/test/src/mill/util/TestUtil.scala index ebe832f3ccc..76278e2fed6 100644 --- a/main/test/src/mill/util/TestUtil.scala +++ b/main/test/src/mill/util/TestUtil.scala @@ -12,7 +12,7 @@ import scala.collection.mutable object TestUtil extends MillTestKit { - override val targetDir = sys.env.get("MILL_TEST_DEST_FOLDER") match{ + override val targetDir = sys.env.get("MILL_TEST_DEST_FOLDER") match { case Some(v) => os.Path(v) case None => os.pwd / "target" } From b4506ccf6d8a25974a0f804ea5b99d4678f4e6e8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 18 Jul 2024 08:05:45 +0800 Subject: [PATCH 10/13] . --- main/eval/src/mill/eval/EvaluatorCore.scala | 12 ++++++++++-- main/eval/src/mill/eval/Terminal.scala | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 6fcf7984861..6110ac7358a 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -150,8 +150,16 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { } } - val (commands, tasks) = terminals0.partition { - case Terminal.Labelled(c: Command[_], _) => true + val tasks0 = terminals0.filter { + case Terminal.Labelled(c: Command[_], _) => false + case _ => true + } + + val (_, tasksTransitive0) = Plan.plan(Agg.from(tasks0.map(_.task))) + + val tasksTransitive = tasksTransitive0.toSet + val (tasks, commands) = terminals0.partition { + case Terminal.Labelled(t, _) if tasksTransitive.contains(t) => true case _ => false } diff --git a/main/eval/src/mill/eval/Terminal.scala b/main/eval/src/mill/eval/Terminal.scala index a8b32658a29..2baf1dc9a98 100644 --- a/main/eval/src/mill/eval/Terminal.scala +++ b/main/eval/src/mill/eval/Terminal.scala @@ -9,6 +9,7 @@ import mill.define.{NamedTask, Segment, Segments} */ sealed trait Terminal { def render: String + def task: mill.define.Task[_] } object Terminal { From 728ceff34e3f2da752fed407ee2be877c302531e Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 18 Jul 2024 08:16:05 +0800 Subject: [PATCH 11/13] . --- build.sc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sc b/build.sc index 20c57b84c4d..9b432c8e219 100644 --- a/build.sc +++ b/build.sc @@ -506,7 +506,9 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { ), ProblemFilter.exclude[ReversedMissingMethodProblem]( "mill.scalalib.JavaModule.mill$scalalib$JavaModule$$super$runMain" - ) + ), + // Terminal is sealed, not sure why MIMA still complains + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.eval.Terminal.task") ) def mimaPreviousVersions: T[Seq[String]] = Settings.mimaBaseVersions From 02b20868638a0a4d6f8c07f805446b8649f7a9c0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 18 Jul 2024 16:58:00 +0800 Subject: [PATCH 12/13] . --- main/eval/src/mill/eval/EvaluatorCore.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 6110ac7358a..e33143749ff 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -158,7 +158,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val (_, tasksTransitive0) = Plan.plan(Agg.from(tasks0.map(_.task))) val tasksTransitive = tasksTransitive0.toSet - val (tasks, commands) = terminals0.partition { + val (tasks, leafCommands) = terminals0.partition { case Terminal.Labelled(t, _) if tasksTransitive.contains(t) => true case _ => false } @@ -167,10 +167,12 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { // given but run the commands in linear order evaluateTerminals( tasks, + // We want to skip the non-deterministic thread prefix in our test suite + // since all it would do is clutter the testing logic trying to match on it if (sys.env.contains("MILL_TEST_SUITE")) _ => "" else contextLoggerMsg0 )(ec) - evaluateTerminals(commands, _ => "")(ExecutionContexts.RunNow) + evaluateTerminals(leafCommands, _ => "")(ExecutionContexts.RunNow) val finishedOptsMap = terminals0 .map(t => (t, Await.result(futures(t), duration.Duration.Inf))) From f619ea50d2edde933f3910a6c7e163ed7c23f7bf Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 7 Aug 2024 23:12:33 -0700 Subject: [PATCH 13/13] . --- build.sc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sc b/build.sc index 53a222546db..fc522d4ca1f 100644 --- a/build.sc +++ b/build.sc @@ -516,7 +516,7 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { "mill.scalalib.JavaModule.mill$scalalib$JavaModule$$super$runMain" ), // Terminal is sealed, not sure why MIMA still complains - ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.eval.Terminal.task") + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.eval.Terminal.task"), // Not sure why mima is picking up this stuff which is private[mill] ProblemFilter.exclude[Problem]("mill.resolve.*.resolve0"),